diff --git a/parseable-backstage-plugin/plugins/parseable-logstream/src/api/ParseableClient.ts b/parseable-backstage-plugin/plugins/parseable-logstream/src/api/ParseableClient.ts new file mode 100644 index 0000000..adbc7ca --- /dev/null +++ b/parseable-backstage-plugin/plugins/parseable-logstream/src/api/ParseableClient.ts @@ -0,0 +1,285 @@ +import { z } from 'zod'; +import { ConfigApi, FetchApi, IdentityApi } from '@backstage/core-plugin-api'; + +// Types for the Parseable API responses +export interface ParseableUserResponse { + username: string; + datasets: string[]; +} + +export interface ParseableSchemaResponse { + name: string; + schema: Record; +} + +// Schema for validating log entries from the query API +const LogEntrySchema = z.object({ + // Common fields from the sample response + address: z.string().optional(), + commit: z.string().optional(), + event_time: z.string().optional(), + event_type: z.string().optional(), + node_type: z.string().optional(), + p_format: z.string().optional(), + p_timestamp: z.string().optional(), + p_user_agent: z.string().optional(), + + // Metrics fields + parseable_deleted_events_ingested: z.number().optional(), + parseable_deleted_events_ingested_size: z.number().optional(), + parseable_events_ingested: z.number().optional(), + parseable_events_ingested_size: z.number().optional(), + parseable_lifetime_events_ingested: z.number().optional(), + parseable_lifetime_events_ingested_size: z.number().optional(), + parseable_staging_files: z.number().optional(), + process_resident_memory_bytes: z.number().optional(), + staging: z.string().optional(), + + // Nested objects + parseable_deleted_storage_size: z.object({ + staging: z.number().optional(), + data: z.number().optional() + }).optional(), + parseable_lifetime_storage_size: z.object({ + staging: z.number().optional(), + data: z.number().optional() + }).optional(), + parseable_storage_size: z.object({ + staging: z.number().optional(), + data: z.number().optional() + }).optional(), + + // Catch-all for other fields + // This allows the schema to accept any additional fields not explicitly defined +}).passthrough(); + +export type LogEntry = z.infer; + +export interface ParseableClientOptions { + fetchApi: FetchApi; + identityApi: IdentityApi; + configApi: ConfigApi; +} + +export class ParseableClient { + private readonly fetchApi: FetchApi; + private readonly identityApi: IdentityApi; + private readonly configApi: ConfigApi; + + constructor(options: ParseableClientOptions) { + this.fetchApi = options.fetchApi; + this.identityApi = options.identityApi; + this.configApi = options.configApi; + } + + // This method is intentionally left empty as we get the base URL directly from the entity annotation + + /** + * Get the authorization header for Parseable API requests + */ + private async getAuthHeader(baseUrl?: string): Promise { + const headers = new Headers(); + + // If this is the demo server, use the default admin/admin credentials + if (baseUrl && baseUrl.includes('demo.parseable.com')) { + // admin:admin in base64 is YWRtaW46YWRtaW4= + headers.set('Authorization', 'Basic YWRtaW46YWRtaW4='); + return headers; + } + + try { + const parseableCredential = this.configApi.getString('parseable.basicAuthCredential'); + headers.set('Authorization', `Basic ${parseableCredential}`); + } catch (e) { + throw new Error('Failed to get Parseable credentials from config. Make sure PARSEABLE_B64_CRED is set.'); + } + + return headers; + } + + /** + * Get user information and available datasets + */ + async getUserInfo(baseUrl: string): Promise { + // Get identity from Backstage (for username) + const identity = await this.identityApi.getBackstageIdentity(); + const username = identity.userEntityRef.split('/')[1]; + + // Get auth headers with the appropriate credentials + const headers = await this.getAuthHeader(baseUrl); + + // Directly fetch the list of datasets using the correct endpoint + const url = `${baseUrl}/api/v1/logstream`; + + try { + const response = await this.fetchApi.fetch(url, { + headers, + }); + + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + throw new Error('Authentication failed. Please check your Parseable credentials.'); + } + throw new Error(`Failed to fetch datasets: ${response.statusText}`); + } + + const data = await response.json(); + + // Validate that we got an array + if (!Array.isArray(data)) { + throw new Error('Invalid response format from Parseable API: expected an array of datasets'); + } + + // Extract dataset names from the response + // The API returns an array of objects with a 'name' property + // Example: [{ "name": "awsmetrics" }, { "name": "pmeta" }, ...] + const datasets = data.map(item => { + if (typeof item === 'object' && item !== null && 'name' in item) { + return item.name; + } + return typeof item === 'string' ? item : ''; + }).filter(Boolean); + + // Return in the expected format + return { + username: username, + datasets: datasets, + }; + } catch (e) { + if (e instanceof Error) { + throw e; + } + throw new Error(`Failed to fetch datasets: ${String(e)}`); + } + } + + /** + * Get schema for a specific dataset + */ + async getSchema(baseUrl: string, dataset: string): Promise { + // Get auth headers with the appropriate credentials + const headers = await this.getAuthHeader(baseUrl); + + // Fetch the schema for the selected dataset + const url = `${baseUrl}/api/v1/logstream/${dataset}/schema`; + + try { + const response = await this.fetchApi.fetch(url, { + headers, + }); + + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + throw new Error('Authentication failed. Please check your Parseable credentials.'); + } + throw new Error(`Failed to fetch schema: ${response.statusText}`); + } + + const data = await response.json(); + + return { + name: dataset, + schema: data, + }; + } catch (e) { + if (e instanceof Error) { + throw e; + } + throw new Error(`Failed to fetch schema: ${String(e)}`); + } + } + + /** + * Get logs for a specific dataset + */ + async getLogs( + baseUrl: string, + dataset: string, + limit: number = 100, + query?: string, + startTime?: string, + endTime?: string + ): Promise { + const headers = await this.getAuthHeader(baseUrl); + const requestHeaders = new Headers(headers); + requestHeaders.set('content-type', 'application/json'); + + // If no time range is provided, default to last 5 minutes + if (!startTime || !endTime) { + const now = new Date(); + endTime = now.toISOString(); + const fiveMinutesAgo = new Date(now.getTime() - 5 * 60 * 1000); + startTime = fiveMinutesAgo.toISOString(); + } + + // Build the SQL query + let sqlQuery = `select * from ${dataset}`; + if (query && query.trim() !== '') { + sqlQuery += ` where ${query}`; + } + + // Add limit to the query if specified + if (limit && limit > 0) { + sqlQuery += ` limit ${limit}`; + } + + const requestBody = { + query: sqlQuery, + startTime: startTime, + endTime: endTime, + }; + + try { + + const response = await this.fetchApi.fetch(`${baseUrl}/api/v1/query`, { + method: 'POST', + headers: requestHeaders, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + throw new Error('Authentication failed. Please check your Parseable credentials.'); + } + throw new Error(`Failed to fetch logs: ${response.statusText}`); + } + + const data = await response.json(); + return Array.isArray(data) ? data.map(entry => LogEntrySchema.parse(entry)) : []; + } catch (e) { + if (e instanceof z.ZodError) { + throw new Error(`Invalid log format from Parseable API: ${e.message}`); + } + throw e; + } + } + + /** + * Export logs to CSV + */ + async exportToCsv(baseUrl: string, dataset: string, query?: string): Promise { + const headers = await this.getAuthHeader(baseUrl); + let url = `${baseUrl}/api/v1/logstream/${dataset}/csv`; + + if (query) { + url += `?q=${encodeURIComponent(query)}`; + } + + try { + const response = await this.fetchApi.fetch(url, { + headers, + }); + + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + throw new Error('Authentication failed. Please check your Parseable credentials.'); + } + throw new Error(`Failed to export logs: ${response.statusText}`); + } + + return await response.blob(); + } catch (e) { + throw new Error(`Failed to export logs to CSV: ${e instanceof Error ? e.message : String(e)}`); + } + } +} diff --git a/parseable-backstage-plugin/plugins/parseable-logstream/src/components/ParseableLogstreamPage.tsx b/parseable-backstage-plugin/plugins/parseable-logstream/src/components/ParseableLogstreamPage.tsx new file mode 100644 index 0000000..6438d4c --- /dev/null +++ b/parseable-backstage-plugin/plugins/parseable-logstream/src/components/ParseableLogstreamPage.tsx @@ -0,0 +1,523 @@ +import React, { useState, useEffect } from 'react'; +import { useApi } from '@backstage/core-plugin-api'; +import { useEntity } from '@backstage/plugin-catalog-react'; +import { + Content, + ContentHeader, + SupportButton, + InfoCard, + Progress, + ErrorPanel, + EmptyState, + Table, +} from '@backstage/core-components'; +import { + Button, + FormControl, + InputLabel, + MenuItem, + Select, + makeStyles, + TextField, + Grid, + Typography, + IconButton, + Tooltip, +} from '@material-ui/core'; +import SearchIcon from '@material-ui/icons/Search'; +import PlayArrowIcon from '@material-ui/icons/PlayArrow'; +import PauseIcon from '@material-ui/icons/Pause'; +import GetAppIcon from '@material-ui/icons/GetApp'; +import { useAsync } from 'react-use'; +import { parseableApiRef } from '../api'; +import type { LogEntry, ParseableSchemaResponse } from '../api/ParseableClient'; +import { error } from 'console'; + +const useStyles = makeStyles(theme => ({ + root: { + display: 'flex', + flexDirection: 'column', + height: '100%', + }, + controls: { + display: 'flex', + flexWrap: 'wrap', + gap: theme.spacing(2), + marginBottom: theme.spacing(2), + }, + formControl: { + minWidth: 200, + }, + searchField: { + minWidth: 300, + }, + logContainer: { + height: 'calc(100vh - 300px)', + overflowY: 'auto', + backgroundColor: theme.palette.background.default, + padding: theme.spacing(1), + fontFamily: 'monospace', + fontSize: '0.8rem', + }, + logLine: { + padding: theme.spacing(0.5), + borderBottom: `1px solid ${theme.palette.divider}`, + display: 'flex', + alignItems: 'center', + '&:hover': { + backgroundColor: theme.palette.action.hover, + }, + }, + logContent: { + flexGrow: 1, + wordBreak: 'break-all', + }, + copyButton: { + visibility: 'hidden', + padding: theme.spacing(0.5), + '$logLine:hover &': { + visibility: 'visible', + }, + }, + timeRangeControls: { + display: 'flex', + gap: theme.spacing(2), + alignItems: 'center', + }, + schemaCard: { + marginBottom: theme.spacing(2), + }, + logsContainer: { + marginTop: theme.spacing(2), + }, + error: { + color: theme.palette.error.main, + }, + warn: { + color: theme.palette.warning.main, + }, + info: { + color: theme.palette.info.main, + }, +})); + +export const ParseableLogstreamPage = () => { + const classes = useStyles(); + const parseableClient = useApi(parseableApiRef); + + const [selectedDataset, setSelectedDataset] = useState(''); + const [logs, setLogs] = useState([]); + const [logsLoading, setLogsLoading] = useState(false); + const [error, setError] = useState(undefined); + const [isLiveTail, setIsLiveTail] = useState(false); + const [searchQuery, setSearchQuery] = useState(''); + const [startDate, setStartDate] = useState(''); + const [endDate, setEndDate] = useState(''); + const [baseUrl, setBaseUrl] = useState('https://demo.parseable.com'); + const [schema, setSchema] = useState(null); + const [schemaLoading, setSchemaLoading] = useState(false); + const [schemaError, setSchemaError] = useState(null); + + // Try to get entity context, but don't fail if not available + const entityContext = (() => { + try { + const { entity } = useEntity(); + return { entity, available: true }; + } catch (e) { + return { entity: undefined, available: false }; + } + })(); + + // Set base URL from entity annotation or allow manual input in standalone mode + useEffect(() => { + if (entityContext.available && entityContext.entity) { + const url = entityContext.entity.metadata.annotations?.['parseable.io/base-url'] || ''; + setBaseUrl(url); + } + }, [entityContext]); + + // Fetch datasets when base URL changes + const { loading: datasetsLoading, value: datasets = [], error: datasetsError } = useAsync( + async () => { + if (!baseUrl) return []; + + try { + // This will use admin/admin credentials for demo.parseable.com + const userInfo = await parseableClient.getUserInfo(baseUrl); + console.log('Fetched datasets:', userInfo.datasets); + return userInfo.datasets || []; + } catch (error) { + console.error('Error fetching datasets:', error); + throw error; + } + }, + [baseUrl] + ); + + // Fetch logs when dataset is selected or during live tail + useEffect(() => { + let intervalId: NodeJS.Timeout; + + const fetchLogs = async () => { + if (!baseUrl || !selectedDataset) return; + + try { + setLogsLoading(true); + // Build query string with search terms + let query = searchQuery; + + // Use the updated getLogs method with time range parameters + const logData = await parseableClient.getLogs( + baseUrl, + selectedDataset, + 100, + query, + startDate || undefined, + endDate || undefined + ); + + setLogs(logData); + setError(undefined); + setLogsLoading(false); + } catch (err) { + console.error('Error fetching logs:', err); + setError(err instanceof Error ? err : new Error(String(err))); + setIsLiveTail(false); + setLogsLoading(false); + } + }; + + // Initial fetch + if (selectedDataset) { + fetchLogs(); + } + + // Set up live tail if enabled + if (isLiveTail && selectedDataset) { + intervalId = setInterval(fetchLogs, 3000); + } + + return () => { + if (intervalId) { + clearInterval(intervalId); + } + }; + }, [baseUrl, selectedDataset, isLiveTail, searchQuery, startDate, endDate, parseableClient]); + + const handleDatasetChange = (event: React.ChangeEvent<{ value: unknown }>) => { + const newDataset = event.target.value as string; + setSelectedDataset(newDataset); + setLogs([]); + setError(undefined); + + // Fetch schema for the selected dataset + if (newDataset && baseUrl) { + setSchemaLoading(true); + setSchema(null); + setSchemaError(null); + + parseableClient.getSchema(baseUrl, newDataset) + .then(response => { + setSchema(response); + setSchemaLoading(false); + }) + .catch(err => { + console.error('Error fetching schema:', err); + setSchemaError(err instanceof Error ? err : new Error(String(err))); + setSchemaLoading(false); + }); + } else { + setSchema(null); + } + }; + + const toggleLiveTail = () => { + setIsLiveTail(!isLiveTail); + }; + + const handleSearch = () => { + // Trigger a search with the current query + setIsLiveTail(false); // Stop live tail when searching + }; + + // Helper function to determine log level color based on log properties + const getLogLevelColor = (log: LogEntry): string => { + if (!log) return 'inherit'; + + // Extract level information from various possible fields + const level = ( + log.level || + log.severity || + log.event_type || + '' + ).toString().toLowerCase(); + + // Check for error indicators + if (level.includes('error')) return '#d32f2f'; // red + if (level.includes('warn') || level.includes('warning')) return '#f57c00'; // orange + if (level.includes('info')) return '#0288d1'; // blue + + // For Parseable metrics, use blue for normal metrics + if (level.includes('metrics')) return '#0288d1'; + + return 'inherit'; + }; + + if (logsLoading) { + return ; + } + + if (error) { + return ( + + + + Check your Parseable credentials and base URL configuration. + + + ); + } + + if (datasets.length === 0) { + return ( + + + + + ); + } + + return ( + + + + View your Parseable log streams with advanced search capabilities. + + + + + + + {!entityContext.available && ( +
+ + You're viewing this plugin in standalone mode. Please enter your Parseable server URL below: + + setBaseUrl(e.target.value)} + placeholder="https://demo.parseable.com" + variant="outlined" + size="small" + disabled={datasetsLoading} + /> +
+ )} + + + Dataset + + + +
+
+ setStartDate(e.target.value)} + InputLabelProps={{ shrink: true }} + disabled={isLiveTail} + /> + + setEndDate(e.target.value)} + InputLabelProps={{ shrink: true }} + disabled={isLiveTail} + /> +
+ + setSearchQuery(e.target.value)} + onKeyPress={(e) => e.key === 'Enter' && handleSearch()} + disabled={isLiveTail} + /> + + + + + + {isLiveTail ? : } + + + + {selectedDataset && ( + + )} +
+ + {error && ( + + )} +
+
+ + {selectedDataset && ( + + {schemaLoading && } + {schemaError && } + {schema && ( + + ({ + field, + type, + }))} + columns={[ + { title: 'Field', field: 'field' }, + { title: 'Type', field: 'type' }, + ]} + /> + + )} + + )} + + + {logsLoading && } + {error && } + {!logsLoading && !error && logs.length === 0 && ( + + )} + {!selectedDataset && ( + + Select a dataset to view logs + + )} + {!logsLoading && !error && logs.length > 0 && ( + +
{ + const level = log.level || log.severity || log.event_type || 'unknown'; + // Get the color for the log level + const levelColor = getLogLevelColor(log); + + return { + id: index, + timestamp: log.p_timestamp || log.event_time || '', + level: level, + message: JSON.stringify(log, null, 2), + // Store the color as a string + levelColor: levelColor + }; + })} + columns={[ + { title: 'Timestamp', field: 'timestamp' }, + { + title: 'Level', + field: 'level', + render: rowData => { + return ( + + {rowData.level} + + ); + } + }, + { + title: 'Message', + field: 'message', + render: rowData => ( +
+                        {rowData.message}
+                      
+ ) + }, + ]} + /> + + )} + + + + ); +};