diff --git a/.phpunit.result.cache b/.phpunit.result.cache index a99474f..8ca16f4 100644 --- a/.phpunit.result.cache +++ b/.phpunit.result.cache @@ -1 +1 @@ -{"version":1,"defects":{"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":3,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":3},"times":{"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalCreation":0.001,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalAllValidIntervals":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidIntervalEmpty":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testIsGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByIntervalReturnsNullWhenMissing":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testRemoveGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testValidIntervalsConstant":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testUsageQueryExtendsQuery":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventSchemaReturnsAttributeDefinitions":0.001,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeSchemaReturnsAttributeDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetSchemaReturnsEventSchema":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIndexesReturnsEventIndexes":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidEventData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidGaugeData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsMinimalData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonStringMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsOversizedMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonIntegerValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDateTimeForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDatetimeStringForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsInvalidDatetimeString":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonArrayTags":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsEmptyTags":0,"Utopia\\Tests\\Usage\\MetricTest::testConstructorInitializesWithData":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsMetricId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsEmptyStringWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetMetricReturnsMetricName":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsType":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsDefaultType":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnCorrectValues":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsTimestamp":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsTags":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsEmptyArrayWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsTenantId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantConvertsNumericToString":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributesReturnsAllAttributes":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSetsAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSupportsChaining":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsTrueWhenExists":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsFalseWhenNotExists":0,"Utopia\\Tests\\Usage\\MetricTest::testRemoveAttributeRemovesAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsFalseWhenIdSet":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsTrueWhenNoId":0,"Utopia\\Tests\\Usage\\MetricTest::testToArrayReturnsArray":0,"Utopia\\Tests\\Usage\\MetricTest::testEventColumnsConstant":0}} \ No newline at end of file +{"version":1,"defects":{"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":3,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":3,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonArrayTags":3},"times":{"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalCreation":0.001,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalAllValidIntervals":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidIntervalEmpty":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testIsGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByIntervalReturnsNullWhenMissing":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testRemoveGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testValidIntervalsConstant":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testUsageQueryExtendsQuery":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventSchemaReturnsAttributeDefinitions":0.001,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeSchemaReturnsAttributeDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetSchemaReturnsEventSchema":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIndexesReturnsEventIndexes":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidEventData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidGaugeData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsMinimalData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonStringMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsOversizedMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonIntegerValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDateTimeForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDatetimeStringForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsInvalidDatetimeString":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonArrayTags":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsEmptyTags":0,"Utopia\\Tests\\Usage\\MetricTest::testConstructorInitializesWithData":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsMetricId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsEmptyStringWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetMetricReturnsMetricName":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsType":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsDefaultType":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnCorrectValues":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsTimestamp":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsTags":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsEmptyArrayWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsTenantId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantConvertsNumericToString":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributesReturnsAllAttributes":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSetsAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSupportsChaining":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsTrueWhenExists":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsFalseWhenNotExists":0,"Utopia\\Tests\\Usage\\MetricTest::testRemoveAttributeRemovesAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsFalseWhenIdSet":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsTrueWhenNoId":0,"Utopia\\Tests\\Usage\\MetricTest::testToArrayReturnsArray":0,"Utopia\\Tests\\Usage\\MetricTest::testEventColumnsConstant":0,"Utopia\\Tests\\Usage\\MetricTest::testGaugeColumnsConstant":0,"Utopia\\Tests\\Usage\\MetricTest::testEventSchemaHasAllNewColumns":0,"Utopia\\Tests\\Usage\\MetricTest::testGaugeSchemaHasTeamAndResourceColumns":0}} \ No newline at end of file diff --git a/README.md b/README.md index eb92d96..c3e5f9b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Although this library is part of the [Utopia Framework](https://github.com/utopi ## Features - **Two Table Architecture**: Separate events and gauges tables optimized for their access patterns -- **Events Table**: Request-level metrics with dedicated columns (path, method, status, resource, country, userAgent) +- **Events Table**: Request-level metrics with dedicated columns for every dimension we filter on (path/method/status, service/resource ids, team ids, country/region, hostname, parsed UA fields) - **Gauges Table**: Simple resource snapshots (storage size, user count, etc.) - **Query-Time Aggregation**: No write-time period fan-out — aggregate by any interval at query time - **Daily Materialized View**: Pre-aggregated daily SummingMergeTree for fast billing queries @@ -72,7 +72,12 @@ $usage->setup(); Events are request-level metrics like bandwidth, executions, API calls. They are summed when aggregated. -Event-specific columns: `path`, `method`, `status`, `resource`, `resourceId`, `country`, `userAgent` +Event-specific columns (see `Metric::EVENT_COLUMNS`): `path`, `method`, `status`, +`service`, `resource`, `resourceId`, `resourceInternalId`, `teamId`, +`teamInternalId`, `country`, `region`, `hostname`, `osCode`, `osName`, +`osVersion`, `clientType`, `clientCode`, `clientName`, `clientVersion`, +`clientEngine`, `clientEngineVersion`, `deviceName`, `deviceBrand`, +`deviceModel`. ```php // Collect events — values accumulate in-memory buffer (summed per metric) @@ -80,26 +85,43 @@ $usage->collect('bandwidth', 5000, Usage::TYPE_EVENT, [ 'path' => '/v1/storage/files', 'method' => 'POST', 'status' => '201', + 'service' => 'storage', 'resource' => 'bucket', 'resourceId' => 'abc123', + 'resourceInternalId' => '42', + 'teamId' => 'team_x', + 'teamInternalId' => '7', 'country' => 'US', - 'userAgent' => 'AppwriteSDK/1.0', + 'region' => 'us-east', + 'hostname' => 'app.example.com', + 'osName' => 'iOS', + 'clientName' => 'Appwrite SDK', + 'deviceName' => 'smartphone', ]); - -// Event columns are auto-extracted from tags into dedicated columns -// Remaining tags stay in the JSON tags column ``` +### Strict tag keys + +All keys passed in the `tags` array must map to a known event or gauge +column (see `Metric::EVENT_COLUMNS` and `Metric::GAUGE_COLUMNS`). Unknown +keys throw at write time — there is no JSON catch-all. To add a new +dimension, widen the schema and bump the library version. + ### Gauges (Point-in-Time) Gauges are resource snapshots like storage size, user count, file count. Last-write-wins semantics. +Gauge-specific columns (see `Metric::GAUGE_COLUMNS`): `teamId`, +`teamInternalId`, `resourceId`, `resourceInternalId`. + ```php // Collect gauges — last value wins per metric in buffer $usage->collect('users', 1500, Usage::TYPE_GAUGE); $usage->collect('storage.size', 1048576, Usage::TYPE_GAUGE, [ - 'resource' => 'bucket', + 'teamId' => 'team_x', + 'teamInternalId' => '7', 'resourceId' => 'abc123', + 'resourceInternalId' => '42', ]); ``` @@ -121,12 +143,12 @@ $usage->setFlushInterval(10); // Flush after 10 seconds (default: 20) ```php // Write directly without buffering $usage->addBatch([ - ['metric' => 'requests', 'value' => 100, 'tags' => ['path' => '/v1/users']], - ['metric' => 'bandwidth', 'value' => 50000, 'tags' => ['country' => 'DE']], + ['metric' => 'requests', 'value' => 100, 'tags' => ['path' => '/v1/users', 'method' => 'GET']], + ['metric' => 'bandwidth', 'value' => 50000, 'tags' => ['country' => 'DE', 'region' => 'fra']], ], Usage::TYPE_EVENT); $usage->addBatch([ - ['metric' => 'users', 'value' => 42, 'tags' => []], + ['metric' => 'users', 'value' => 42, 'tags' => ['teamId' => 'team_x']], ], Usage::TYPE_GAUGE); ``` @@ -250,11 +272,21 @@ $usage->purge([], Usage::TYPE_GAUGE); | path | Nullable(String) | API endpoint path | | method | Nullable(String) | HTTP method | | status | Nullable(String) | HTTP status code | -| resource | Nullable(String) | Resource type | -| resourceId | Nullable(String) | Resource ID | -| country | LowCardinality(Nullable(String)) | ISO country code | -| userAgent | Nullable(String) | User agent string | -| tags | Nullable(String) | JSON for extra metadata | +| service | LowCardinality(Nullable(String)) | API service (storage, databases, …) | +| resource | LowCardinality(Nullable(String)) | Resource type (bucket, file, …) | +| resourceId | Nullable(String) | External resource id | +| resourceInternalId | Nullable(String) | Internal resource sequence | +| teamId | Nullable(String) | External team id | +| teamInternalId | Nullable(String) | Internal team sequence | +| country | LowCardinality(Nullable(String)) | ISO country code (lowercased) | +| region | LowCardinality(Nullable(String)) | Region code (lowercased) | +| hostname | Nullable(String) | Caller origin host | +| osCode, osName | LowCardinality(Nullable(String)) | Parsed OS short code / name | +| osVersion | Nullable(String) | Parsed OS version | +| clientType, clientCode, clientName, clientEngine | LowCardinality(Nullable(String)) | Parsed client identity | +| clientVersion, clientEngineVersion | Nullable(String) | Parsed client versions | +| deviceName, deviceBrand | LowCardinality(Nullable(String)) | Parsed device identity | +| deviceModel | Nullable(String) | Parsed device model | | tenant | Nullable(String) | Tenant ID (shared tables) | ### Gauges Table Schema @@ -265,7 +297,10 @@ $usage->purge([], Usage::TYPE_GAUGE); | metric | String | Metric name | | value | Int64 | Current value | | time | DateTime64(3) | Snapshot timestamp | -| tags | Nullable(String) | JSON metadata | +| teamId | Nullable(String) | External team id | +| teamInternalId | Nullable(String) | Internal team sequence | +| resourceId | Nullable(String) | External resource id | +| resourceInternalId | Nullable(String) | Internal resource sequence | | tenant | Nullable(String) | Tenant ID (shared tables) | ### Daily Table Schema @@ -275,6 +310,11 @@ $usage->purge([], Usage::TYPE_GAUGE); | metric | String | Metric name | | value | Int64 | Aggregated daily sum | | time | DateTime64(3) | Day start timestamp | +| resource | LowCardinality(Nullable(String)) | Resource type | +| resourceId | Nullable(String) | External resource id | +| resourceInternalId | Nullable(String) | Internal resource sequence | +| teamId | Nullable(String) | External team id | +| teamInternalId | Nullable(String) | Internal team sequence | | tenant | Nullable(String) | Tenant ID (shared tables) | ### Creating Custom Adapters diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index f0f67dc..ad7a33a 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1072,6 +1072,11 @@ private function createDailyTable(): void 'metric String', 'value Int64', 'time DateTime64(3)', + 'resource LowCardinality(Nullable(String))', + 'resourceId Nullable(String)', + 'resourceInternalId Nullable(String)', + 'teamId Nullable(String)', + 'teamInternalId Nullable(String)', ]; if ($this->sharedTables) { @@ -1080,9 +1085,9 @@ private function createDailyTable(): void $columnDefs = implode(",\n ", $columns); - // metric and time are part of the ORDER BY (primary key) — no - // secondary bloom_filter indexes needed. - $dailyOrderBy = $this->sharedTables ? '(tenant, metric, time)' : '(metric, time)'; + $dailyOrderBy = $this->sharedTables + ? '(tenant, metric, time, resource, resourceId, resourceInternalId, teamId, teamInternalId)' + : '(metric, time, resource, resourceId, resourceInternalId, teamId, teamInternalId)'; $createDailyTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDailyTable} ( @@ -1113,17 +1118,19 @@ private function createDailyMaterializedView(): void $dailyMvName = $this->getTableName() . '_events_daily_mv'; $escapedEventsTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($eventsTable); - $escapedDailyTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyTableName); - $escapedDailyMv = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyMvName); + $escapedDailyTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyTableName); + $escapedDailyMv = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyMvName); + + $dimensions = 'resource, resourceId, resourceInternalId, teamId, teamInternalId'; if ($this->sharedTables) { - $innerSelect = "metric, tenant, sum(value) as value, toStartOfDay(time) as d"; - $innerGroupBy = "metric, tenant, d"; - $outerSelect = "metric, value, d as time, tenant"; + $innerSelect = "metric, tenant, {$dimensions}, sum(value) as value, toStartOfDay(time) as d"; + $innerGroupBy = "metric, tenant, {$dimensions}, d"; + $outerSelect = "metric, value, d as time, tenant, {$dimensions}"; } else { - $innerSelect = "metric, sum(value) as value, toStartOfDay(time) as d"; - $innerGroupBy = "metric, d"; - $outerSelect = "metric, value, d as time"; + $innerSelect = "metric, {$dimensions}, sum(value) as value, toStartOfDay(time) as d"; + $innerGroupBy = "metric, {$dimensions}, d"; + $outerSelect = "metric, value, d as time, {$dimensions}"; } $createDailyMvSql = " @@ -1172,10 +1179,35 @@ private function validateAttributeName(string $attributeName, string $type = 'ev throw new Exception("Invalid attribute name: {$attributeName}"); } + /** + * Validate that a groupBy attribute is an aggregable dimension column. + * + * Restricted to the indexed dimension columns for the table type — `metric`, + * `value` and `time` are excluded since they are already part of the + * aggregation (metric is in the SELECT, time is bucketed via + * groupByInterval, value is the measured quantity). + * + * @throws Exception + */ + private function validateGroupByAttribute(string $attribute, string $type): bool + { + $allowed = $type === Usage::TYPE_GAUGE ? Metric::GAUGE_COLUMNS : Metric::EVENT_COLUMNS; + + if (in_array($attribute, $allowed, true)) { + return true; + } + + throw new Exception("Invalid groupBy attribute '{$attribute}' for {$type}. Allowed: " . implode(', ', $allowed)); + } + /** * Columns available in the events daily (pre-aggregated) table. */ - private const DAILY_COLUMNS = ['metric', 'value', 'time']; + private const DAILY_COLUMNS = [ + 'metric', 'value', 'time', + 'resource', 'resourceId', 'resourceInternalId', + 'teamId', 'teamInternalId', + ]; /** * Validate that a query attribute exists in the daily table schema. @@ -1197,9 +1229,10 @@ private function validateDailyAttributeName(string $attributeName): bool return true; } + $allowed = implode(', ', self::DAILY_COLUMNS) . ($this->sharedTables ? ', tenant' : ''); throw new Exception( "Invalid attribute '{$attributeName}' for daily table. " - . "Only metric, value, time" . ($this->sharedTables ? ", tenant" : "") . " are available." + . "Allowed: {$allowed}." ); } @@ -1248,8 +1281,13 @@ private function getColumnType(string $id, string $type = 'event'): string throw new Exception("Attribute {$id} not found in {$type} schema"); } - // Country uses LowCardinality for efficient storage of low-cardinality values - if ($id === 'country') { + $lowCardinality = [ + 'country', 'region', 'service', 'resource', + 'osCode', 'osName', 'clientType', 'clientCode', 'clientName', + 'clientEngine', 'deviceName', 'deviceBrand', + ]; + + if (in_array($id, $lowCardinality, true)) { return 'LowCardinality(Nullable(String))'; } @@ -1388,41 +1426,14 @@ public function addBatch(array $metrics, string $type, int $batchSize = self::IN $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; - if ($type === Usage::TYPE_EVENT) { - // Extract event-specific columns from tags into dedicated columns - $eventColumns = []; - foreach (Metric::EVENT_COLUMNS as $col) { - if (isset($tags[$col])) { - $tagValue = $tags[$col]; - $eventColumns[$col] = is_string($tagValue) ? $tagValue : (is_scalar($tagValue) ? (string) $tagValue : null); - unset($tags[$col]); - } else { - $eventColumns[$col] = null; - } - } - - ksort($tags); + $columns = Metric::extractColumns($tags, $type); - $row = array_merge([ - 'id' => $this->generateId(), - 'metric' => $metric, - 'value' => $value, - 'time' => $this->formatDateTime(null), - ], $eventColumns, [ - 'tags' => $tags, - ]); - } else { - // Gauge: simple schema - ksort($tags); - - $row = [ - 'id' => $this->generateId(), - 'metric' => $metric, - 'value' => $value, - 'time' => $this->formatDateTime(null), - 'tags' => $tags, - ]; - } + $row = array_merge([ + 'id' => $this->generateId(), + 'metric' => $metric, + 'value' => $value, + 'time' => $this->formatDateTime(null), + ], $columns); if ($this->sharedTables) { $row['tenant'] = $tenant; @@ -1577,6 +1588,10 @@ private function findFromTable(array $queries, string $type): array throw new Exception('Cursor pagination cannot be combined with groupByInterval'); } + if (!empty($parsed['groupBy']) && !isset($parsed['groupByInterval'])) { + throw new Exception('groupBy requires groupByInterval to be specified'); + } + // Check if groupByInterval is requested if (isset($parsed['groupByInterval'])) { return $this->findAggregatedFromTable($parsed, $fromTable, $type); @@ -1639,7 +1654,7 @@ private function findFromTable(array $queries, string $type): array * toStartOfInterval(time, INTERVAL 1 HOUR) as time * FROM table WHERE ... GROUP BY metric, time ORDER BY time ASC * - * @param array{filters: array, params: array, orderBy?: array, limit?: int, offset?: int, groupByInterval?: string} $parsed Parsed query data from parseQueries() + * @param array{filters: array, params: array, orderBy?: array, limit?: int, offset?: int, groupByInterval?: string, groupBy?: array} $parsed Parsed query data from parseQueries() * @param string $fromTable Fully qualified table reference * @param string $type 'event' or 'gauge' * @return array @@ -1660,6 +1675,18 @@ private function findAggregatedFromTable(array $parsed, string $fromTable, strin // then alias back to 'time' in outer context for consistent Metric parsing. $timeBucketExpr = "toStartOfInterval(time, {$intervalSql})"; + $groupByDims = $parsed['groupBy'] ?? []; + $dimSelect = ''; + $dimGroup = ''; + if (!empty($groupByDims)) { + $escapedDims = array_map( + fn (string $dim): string => $this->escapeIdentifier($dim), + $groupByDims + ); + $dimSelect = ', ' . implode(', ', $escapedDims); + $dimGroup = ', ' . implode(', ', $escapedDims); + } + $whereData = $this->buildWhereClause($parsed['filters'], $parsed['params']); $whereClause = $whereData['clause']; $params = $whereData['params']; @@ -1686,9 +1713,9 @@ private function findAggregatedFromTable(array $parsed, string $fromTable, strin $offsetClause = isset($parsed['offset']) ? ' OFFSET {offset:UInt64}' : ''; $sql = " - SELECT metric, {$valueExpr}, {$timeBucketExpr} as bucket + SELECT metric, {$valueExpr}, {$timeBucketExpr} as bucket{$dimSelect} FROM {$fromTable}{$whereClause} - GROUP BY metric, bucket{$orderClause}{$limitClause}{$offsetClause} + GROUP BY metric, bucket{$dimGroup}{$orderClause}{$limitClause}{$offsetClause} FORMAT JSON "; @@ -2774,7 +2801,7 @@ private function buildOrderBySql(array $orderAttributes, bool $flip = false): ar * * @param array $queries * @param string $type 'event' or 'gauge' — used for attribute validation - * @return array{filters: array, params: array, orderBy?: array, orderAttributes?: array, limit?: int, offset?: int, groupByInterval?: string, cursor?: array, cursorDirection?: string} + * @return array{filters: array, params: array, orderBy?: array, orderAttributes?: array, limit?: int, offset?: int, groupByInterval?: string, groupBy?: array, cursor?: array, cursorDirection?: string} * @throws Exception */ private function parseQueries(array $queries, string $type = 'event'): array @@ -2786,6 +2813,7 @@ private function parseQueries(array $queries, string $type = 'event'): array $limit = null; $offset = null; $groupByInterval = null; + $groupBy = []; $cursor = null; $cursorDirection = null; $paramCounter = 0; @@ -3021,6 +3049,13 @@ private function parseQueries(array $queries, string $type = 'event'): array } $groupByInterval = $interval; break; + + case UsageQuery::TYPE_GROUP_BY: + $this->validateGroupByAttribute($attribute, $type); + if (!in_array($attribute, $groupBy, true)) { + $groupBy[] = $attribute; + } + break; } } @@ -3046,6 +3081,10 @@ private function parseQueries(array $queries, string $type = 'event'): array $result['groupByInterval'] = $groupByInterval; } + if (!empty($groupBy)) { + $result['groupBy'] = $groupBy; + } + if ($cursor !== null && $cursorDirection !== null) { $result['cursor'] = $cursor; $result['cursorDirection'] = $cursorDirection; diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 0f5e204..f0b8b29 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -73,7 +73,10 @@ public function setup(): void throw new \Exception('You need to create the database before running Usage setup'); } - // Use event attributes which is a superset (includes path/method/status/resource/resourceId) + // Event schema is a superset of the gauge schema for the dimensions + // that exist in both (resourceId, resourceInternalId, teamId, + // teamInternalId), so a single Database collection backed by the + // event schema works for both types. $attributes = $this->getAttributeDocuments('event'); $indexDocs = $this->getIndexDocuments('event'); @@ -138,25 +141,19 @@ public function addBatch(array $metrics, string $type, int $batchSize = 1000): b throw new \InvalidArgumentException('Value cannot be negative'); } + /** @var array $tags */ $tags = $metric['tags'] ?? []; - ksort($tags); - $docData = [ + $columns = Metric::extractColumns($tags, $type); + + $docData = array_merge([ '$id' => $this->generateId(), '$permissions' => [], 'metric' => $metric['metric'], 'value' => $metric['value'], 'type' => $type, 'time' => (new \DateTime())->format('Y-m-d H:i:s.v'), - 'tags' => $tags, - ]; - - // For events, extract event-specific columns from tags - if ($type === Usage::TYPE_EVENT) { - foreach (Metric::EVENT_COLUMNS as $col) { - $docData[$col] = $tags[$col] ?? null; - } - } + ], $columns); $documents[] = new Document($docData); } @@ -370,9 +367,13 @@ public function sumDaily(array $queries = [], string $attribute = 'value'): int * * @param array $queries * @return array + * @throws \Exception When a groupBy attribute is not a valid dimension column, + * or when groupBy is used without groupByInterval. */ private function convertQueriesToDatabase(array $queries): array { + $this->validateGroupByQueries($queries); + $dbQueries = []; foreach ($queries as $query) { $method = $query->getMethod(); @@ -477,8 +478,10 @@ private function convertQueriesToDatabase(array $queries): array break; case UsageQuery::TYPE_GROUP_BY_INTERVAL: - // groupByInterval is not supported by the Database adapter. - // Silently skip — callers get raw (non-aggregated) results. + case UsageQuery::TYPE_GROUP_BY: + // groupByInterval and groupBy are not pushed down to the + // Database adapter; callers get raw (non-aggregated) results. + // Validation runs in validateGroupByQueries() before this loop. break; } } @@ -486,6 +489,51 @@ private function convertQueriesToDatabase(array $queries): array return $dbQueries; } + /** + * Validate groupBy / groupByInterval interactions in the supplied queries. + * + * Mirrors the ClickHouse adapter contract: groupBy attributes must exist on + * the matching schema (event vs gauge — we default to the broader event set + * for the Database adapter since both share one collection), and groupBy + * must always be paired with groupByInterval so the cloud-facing API stays + * consistent across backends. + * + * @param array $queries + * @throws \Exception + */ + private function validateGroupByQueries(array $queries): void + { + $hasGroupBy = false; + $hasGroupByInterval = false; + $allowed = array_unique(array_merge(Metric::EVENT_COLUMNS, Metric::GAUGE_COLUMNS)); + + foreach ($queries as $query) { + $method = $query->getMethod(); + + if ($method === UsageQuery::TYPE_GROUP_BY_INTERVAL) { + $hasGroupByInterval = true; + continue; + } + + if ($method !== UsageQuery::TYPE_GROUP_BY) { + continue; + } + + $hasGroupBy = true; + $attribute = $query->getAttribute(); + + if (!in_array($attribute, $allowed, true)) { + throw new \Exception( + "Invalid groupBy attribute '{$attribute}'. Allowed: " . implode(', ', $allowed) + ); + } + } + + if ($hasGroupBy && !$hasGroupByInterval) { + throw new \Exception('groupBy requires groupByInterval to be specified'); + } + } + /** * @param array $queries * @param string|null $type diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index 3f26450..e0ab988 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -23,9 +23,18 @@ * 'path' => '/v1/storage/files', * 'method' => 'POST', * 'status' => '201', + * 'service' => 'storage', * 'resource' => 'bucket', * 'resourceId' => 'abc123', - * 'tags' => ['region' => 'us-east', 'country' => 'US'] + * 'resourceInternalId' => '42', + * 'teamId' => 'team_x', + * 'teamInternalId' => '7', + * 'country' => 'us', + * 'region' => 'us-east', + * 'hostname' => 'app.example.com', + * 'osName' => 'iOS', + * 'clientName' => 'Appwrite SDK', + * 'deviceName' => 'smartphone', * ]); * * echo $metric->getMetric(); // 'bandwidth' @@ -40,7 +49,21 @@ class Metric extends ArrayObject /** * Event-specific column names that are extracted from tags into dedicated columns. */ - public const EVENT_COLUMNS = ['path', 'method', 'status', 'resource', 'resourceId', 'country', 'userAgent']; + public const EVENT_COLUMNS = [ + 'path', 'method', 'status', + 'service', 'resource', 'resourceId', 'resourceInternalId', + 'teamId', 'teamInternalId', + 'country', 'region', 'hostname', + 'osCode', 'osName', 'osVersion', + 'clientType', 'clientCode', 'clientName', 'clientVersion', + 'clientEngine', 'clientEngineVersion', + 'deviceName', 'deviceBrand', 'deviceModel', + ]; + + /** + * Gauge-specific column names that are extracted from tags into dedicated columns. + */ + public const GAUGE_COLUMNS = ['teamId', 'teamInternalId', 'resourceId', 'resourceInternalId']; /** * Construct a new metric object. @@ -51,14 +74,22 @@ class Metric extends ArrayObject * - metric: Name/type of the metric being tracked * - value: Numeric value of the metric * - time: Timestamp when the metric was recorded - * - path: API endpoint path (events only) - * - method: HTTP method (events only) - * - status: HTTP status code (events only) - * - resource: Resource type (events only) - * - resourceId: Resource ID (events only) - * - tags: Additional metadata as key-value pairs * - tenant: Tenant ID for multi-tenant environments * + * Event-only dimension columns (see EVENT_COLUMNS): + * - path / method / status: HTTP shape + * - service: API service segment (storage, databases, …) + * - resource / resourceId / resourceInternalId: resource identity + * - teamId / teamInternalId: owning team identity + * - country / region / hostname: geographic + caller origin + * - osCode / osName / osVersion: parsed user-agent OS fields + * - clientType / clientCode / clientName / clientVersion: parsed client + * - clientEngine / clientEngineVersion: parsed client engine + * - deviceName / deviceBrand / deviceModel: parsed device fields + * + * Gauge-only dimension columns (see GAUGE_COLUMNS): + * - teamId / teamInternalId / resourceId / resourceInternalId + * * @param array $input Metric data */ public function __construct(array $input = []) @@ -217,41 +248,169 @@ public function getCountry(): ?string } /** - * Get user agent (event metrics only). + * Get service (event metrics only). * - * @return string|null The user agent string, or null if not set + * @return string|null */ - public function getUserAgent(): ?string + public function getService(): ?string { - $userAgent = $this->getAttribute('userAgent', null); - return is_string($userAgent) ? $userAgent : null; + $v = $this->getAttribute('service', null); + return is_string($v) ? $v : null; } /** - * Get tags. - * - * Returns additional metadata associated with this metric as key-value pairs. - * Tags are useful for filtering, grouping, and contextualizing metrics. - * - * Common tag examples: - * - region: Geographic region (us-east, eu-west) - * - userAgent: Client user agent - * - country: Country code + * Get internal resource id (event/gauge metrics). * - * Note: For event metrics, path/method/status/resource/resourceId are stored - * as dedicated columns, not in tags. Remaining metadata (region, userAgent, etc.) - * stays in the tags JSON. - * - * @return array Associative array of tags + * @return string|null + */ + public function getResourceInternalId(): ?string + { + $v = $this->getAttribute('resourceInternalId', null); + return is_string($v) ? $v : null; + } + + /** + * Get team id (event/gauge metrics). */ - // NOTE: loks0n flagged this as a leftover from the previous Metric - // implementation. Kept because tests (MetricTest, ClickHouseTest) and - // downstream consumers still call it; remove once those callers are - // migrated to direct `tags` attribute access. - public function getTags(): array + public function getTeamId(): ?string { - $tags = $this->getAttribute('tags', []); - return is_array($tags) ? $tags : []; + $v = $this->getAttribute('teamId', null); + return is_string($v) ? $v : null; + } + + /** + * Get team internal id (event/gauge metrics). + */ + public function getTeamInternalId(): ?string + { + $v = $this->getAttribute('teamInternalId', null); + return is_string($v) ? $v : null; + } + + /** + * Get region (event metrics). + */ + public function getRegion(): ?string + { + $v = $this->getAttribute('region', null); + return is_string($v) ? $v : null; + } + + /** + * Get caller hostname (event metrics). + */ + public function getHostname(): ?string + { + $v = $this->getAttribute('hostname', null); + return is_string($v) ? $v : null; + } + + /** + * Get OS short code (event metrics). + */ + public function getOsCode(): ?string + { + $v = $this->getAttribute('osCode', null); + return is_string($v) ? $v : null; + } + + /** + * Get OS name (event metrics). + */ + public function getOsName(): ?string + { + $v = $this->getAttribute('osName', null); + return is_string($v) ? $v : null; + } + + /** + * Get OS version (event metrics). + */ + public function getOsVersion(): ?string + { + $v = $this->getAttribute('osVersion', null); + return is_string($v) ? $v : null; + } + + /** + * Get client type (event metrics). + */ + public function getClientType(): ?string + { + $v = $this->getAttribute('clientType', null); + return is_string($v) ? $v : null; + } + + /** + * Get client short code (event metrics). + */ + public function getClientCode(): ?string + { + $v = $this->getAttribute('clientCode', null); + return is_string($v) ? $v : null; + } + + /** + * Get client name (event metrics). + */ + public function getClientName(): ?string + { + $v = $this->getAttribute('clientName', null); + return is_string($v) ? $v : null; + } + + /** + * Get client version (event metrics). + */ + public function getClientVersion(): ?string + { + $v = $this->getAttribute('clientVersion', null); + return is_string($v) ? $v : null; + } + + /** + * Get client engine (event metrics). + */ + public function getClientEngine(): ?string + { + $v = $this->getAttribute('clientEngine', null); + return is_string($v) ? $v : null; + } + + /** + * Get client engine version (event metrics). + */ + public function getClientEngineVersion(): ?string + { + $v = $this->getAttribute('clientEngineVersion', null); + return is_string($v) ? $v : null; + } + + /** + * Get device name (event metrics). + */ + public function getDeviceName(): ?string + { + $v = $this->getAttribute('deviceName', null); + return is_string($v) ? $v : null; + } + + /** + * Get device brand (event metrics). + */ + public function getDeviceBrand(): ?string + { + $v = $this->getAttribute('deviceBrand', null); + return is_string($v) ? $v : null; + } + + /** + * Get device model (event metrics). + */ + public function getDeviceModel(): ?string + { + $v = $this->getAttribute('deviceModel', null); + return is_string($v) ? $v : null; } /** @@ -408,6 +567,16 @@ public function toArray(): array */ public static function getEventSchema(): array { + $stringColumn = static fn (string $id, int $size): array => [ + '$id' => $id, + 'type' => 'string', + 'size' => $size, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ]; + return [ [ '$id' => 'metric', @@ -437,78 +606,30 @@ public static function getEventSchema(): array 'array' => false, 'filters' => ['datetime'], ], - [ - '$id' => 'path', - 'type' => 'string', - 'size' => 255, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'method', - 'type' => 'string', - 'size' => 16, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'status', - 'type' => 'string', - 'size' => 16, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'resource', - 'type' => 'string', - 'size' => 255, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'resourceId', - 'type' => 'string', - 'size' => 255, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'country', - 'type' => 'string', - 'size' => 2, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'userAgent', - 'type' => 'string', - 'size' => 255, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'tags', - 'type' => 'string', - 'size' => 16777216, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => ['json'], - ], + $stringColumn('path', 1024), + $stringColumn('method', 16), + $stringColumn('status', 16), + $stringColumn('service', 256), + $stringColumn('resource', 256), + $stringColumn('resourceId', 255), + $stringColumn('resourceInternalId', 255), + $stringColumn('teamId', 255), + $stringColumn('teamInternalId', 255), + $stringColumn('country', 2), + $stringColumn('region', 64), + $stringColumn('hostname', 255), + $stringColumn('osCode', 256), + $stringColumn('osName', 256), + $stringColumn('osVersion', 255), + $stringColumn('clientType', 256), + $stringColumn('clientCode', 256), + $stringColumn('clientName', 256), + $stringColumn('clientVersion', 255), + $stringColumn('clientEngine', 256), + $stringColumn('clientEngineVersion', 255), + $stringColumn('deviceName', 256), + $stringColumn('deviceBrand', 256), + $stringColumn('deviceModel', 255), ]; } @@ -522,6 +643,16 @@ public static function getEventSchema(): array */ public static function getGaugeSchema(): array { + $stringColumn = static fn (string $id, int $size): array => [ + '$id' => $id, + 'type' => 'string', + 'size' => $size, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ]; + return [ [ '$id' => 'metric', @@ -551,15 +682,10 @@ public static function getGaugeSchema(): array 'array' => false, 'filters' => ['datetime'], ], - [ - '$id' => 'tags', - 'type' => 'string', - 'size' => 16777216, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => ['json'], - ], + $stringColumn('teamId', 255), + $stringColumn('teamInternalId', 255), + $stringColumn('resourceId', 255), + $stringColumn('resourceInternalId', 255), ]; } @@ -583,46 +709,34 @@ public static function getSchema(): array */ public static function getEventIndexes(): array { - // Note: `metric` and `time` are part of the ClickHouse primary key - // (ORDER BY (tenant, metric, time, id)), so a separate bloom_filter - // index on either column would be redundant. They're omitted here. - return [ - [ - '$id' => 'index-path', - 'type' => 'key', - 'attributes' => ['path'], - ], - [ - '$id' => 'index-method', - 'type' => 'key', - 'attributes' => ['method'], - ], - [ - '$id' => 'index-status', - 'type' => 'key', - 'attributes' => ['status'], - ], - [ - '$id' => 'index-resource', - 'type' => 'key', - 'attributes' => ['resource'], - ], - [ - '$id' => 'index-resourceId', - 'type' => 'key', - 'attributes' => ['resourceId'], - ], - [ - '$id' => 'index-country', - 'type' => 'key', - 'attributes' => ['country'], - ], - [ - '$id' => 'index-userAgent', - 'type' => 'key', - 'attributes' => ['userAgent'], - ], + // `metric` and `time` are part of the ClickHouse primary key + // (ORDER BY (tenant, metric, time, id)), so separate bloom_filter + // indexes on them would be redundant. + $indexed = [ + 'path', 'method', 'status', + 'service', 'resource', 'resourceId', 'resourceInternalId', + 'teamId', 'teamInternalId', + 'country', 'region', 'hostname', + 'osName', 'clientType', 'clientName', 'deviceName', ]; + + return array_map( + static function (string $col): array { + $entry = [ + '$id' => 'index-' . $col, + 'type' => 'key', + 'attributes' => [$col], + ]; + // path is sized 1024 for data fidelity; cap the index key at + // 255 so the MariaDB single-attribute index stays within the + // 768-byte InnoDB key prefix limit. + if ($col === 'path') { + $entry['lengths'] = [255]; + } + return $entry; + }, + $indexed, + ); } /** @@ -632,10 +746,14 @@ public static function getEventIndexes(): array */ public static function getGaugeIndexes(): array { - // `metric` and `time` are part of the primary key; no secondary - // indexes needed. Returning an empty array keeps the table DDL - // free of redundant bloom_filter clauses. - return []; + return array_map( + static fn (string $col): array => [ + '$id' => 'index-' . $col, + 'type' => 'key', + 'attributes' => [$col], + ], + ['resourceId', 'resourceInternalId', 'teamId', 'teamInternalId'], + ); } /** @@ -651,6 +769,49 @@ public static function getIndexes(): array return self::getEventIndexes(); } + /** + * Extract and normalize dimension columns from a tags array. + * + * For the given metric type ('event' or 'gauge'): + * - Pulls every known column out of $tags. + * - Coerces scalars to string, empty string to null. + * - Lowercases country and region. + * - Throws if $tags contains any unknown key (strict — no JSON catch-all). + * + * @param array $tags + * @param string $type 'event' or 'gauge' + * @return array + * @throws \Exception When an unknown column key is present in $tags. + */ + public static function extractColumns(array $tags, string $type): array + { + $allowed = $type === 'gauge' ? self::GAUGE_COLUMNS : self::EVENT_COLUMNS; + + $columns = []; + foreach ($allowed as $col) { + $val = $tags[$col] ?? null; + unset($tags[$col]); + if (is_string($val)) { + $val = $val === '' ? null : $val; + } elseif (is_scalar($val)) { + $val = (string) $val; + } else { + $val = null; + } + if (($col === 'country' || $col === 'region') && is_string($val)) { + $val = strtolower($val); + } + $columns[$col] = $val; + } + + if (!empty($tags)) { + $unknown = array_key_first($tags); + throw new \Exception("Unknown column '{$unknown}' for {$type}"); + } + + return $columns; + } + /** * Validate metric data against schema. * @@ -689,14 +850,6 @@ public static function validate(array $data, string $type = 'event'): void $value = $data[$attrId]; - // Special handling for tags: accept array (will be JSON-encoded) - if ($attrId === 'tags') { - if (!is_array($value)) { - throw new \Exception("Attribute '{$attrId}' must be an array, got " . gettype($value)); - } - continue; - } - // Validate based on attribute type match ($attrType) { 'string' => self::validateStringAttribute($attrId, $value, $size), diff --git a/src/Usage/UsageQuery.php b/src/Usage/UsageQuery.php index 586f922..60ce843 100644 --- a/src/Usage/UsageQuery.php +++ b/src/Usage/UsageQuery.php @@ -30,6 +30,7 @@ class UsageQuery extends Query { public const TYPE_GROUP_BY_INTERVAL = 'groupByInterval'; + public const TYPE_GROUP_BY = 'groupBy'; /** * Valid interval values and their ClickHouse INTERVAL equivalents. @@ -45,11 +46,11 @@ class UsageQuery extends Query ]; /** - * Override isMethod to accept groupByInterval in addition to all base Query methods. + * Override isMethod to accept groupByInterval and groupBy in addition to all base Query methods. */ public static function isMethod(string $value): bool { - if ($value === self::TYPE_GROUP_BY_INTERVAL) { + if ($value === self::TYPE_GROUP_BY_INTERVAL || $value === self::TYPE_GROUP_BY) { return true; } @@ -122,4 +123,60 @@ public static function removeGroupByInterval(array $queries): array return !self::isGroupByInterval($query); })); } + + /** + * Create a groupBy query for dimensional aggregation. + * + * Buckets results by the given attribute in addition to the time bucket + * supplied via `groupByInterval`. Multiple `groupBy` queries may be + * combined to bucket by several dimensions at once (e.g. service x status). + * + * @param string $attribute The dimension column to bucket on (service, path, status, ...). + * @return self + */ + public static function groupBy(string $attribute): self + { + return new self(self::TYPE_GROUP_BY, $attribute, []); + } + + /** + * Check if a query is a groupBy query. + * + * @param Query $query + * @return bool + */ + public static function isGroupBy(Query $query): bool + { + return $query->getMethod() === self::TYPE_GROUP_BY; + } + + /** + * Extract all groupBy queries from an array of queries. + * + * Multiple groupBy queries can coexist (group by service AND status), so + * this returns every match rather than the single-instance form used by + * groupByInterval. + * + * @param array $queries + * @return array + */ + public static function extractGroupBy(array $queries): array + { + return array_values(array_filter($queries, function (Query $query) { + return self::isGroupBy($query); + })); + } + + /** + * Remove all groupBy queries from an array of queries. + * + * @param array $queries + * @return array + */ + public static function removeGroupBy(array $queries): array + { + return array_values(array_filter($queries, function (Query $query) { + return !self::isGroupBy($query); + })); + } } diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index 81d3d14..66b5d3d 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -129,7 +129,7 @@ public function testLargeBatchWithSmallBatchSize(): void $metrics[] = [ 'metric' => 'large-batch-metric', 'value' => $i, - 'tags' => ['index' => (string) $i], + 'tags' => ['resourceId' => (string) $i], ]; } @@ -210,7 +210,9 @@ public function testBatchWithTagsClickHouse(): void } /** - * Test event-specific columns are extracted from tags + * Test event-specific columns are extracted from tags into dedicated columns + * and surfaced via the new typed getters. Verifies the full dimension set + * round-trips through ClickHouse. */ public function testEventColumnsExtractedFromTags(): void { @@ -224,11 +226,27 @@ public function testEventColumnsExtractedFromTags(): void 'path' => '/v1/storage/files', 'method' => 'POST', 'status' => '201', + 'service' => 'storage', 'resource' => 'bucket', 'resourceId' => 'bucket123', + 'resourceInternalId' => '42', + 'teamId' => 'team_x', + 'teamInternalId' => '7', 'country' => 'US', - 'userAgent' => 'test-agent', 'region' => 'us-east', + 'hostname' => 'app.example.com', + 'osCode' => 'IOS', + 'osName' => 'iOS', + 'osVersion' => '17.4', + 'clientType' => 'mobile-app', + 'clientCode' => 'APW', + 'clientName' => 'Appwrite SDK', + 'clientVersion' => '15.0.0', + 'clientEngine' => 'WebKit', + 'clientEngineVersion' => '605', + 'deviceName' => 'smartphone', + 'deviceBrand' => 'Apple', + 'deviceModel' => 'iPhone 13', ], ], ]; @@ -242,25 +260,155 @@ public function testEventColumnsExtractedFromTags(): void $this->assertCount(1, $results); $metric = $results[0]; - // Event-specific columns should be set $this->assertEquals('/v1/storage/files', $metric->getPath()); $this->assertEquals('POST', $metric->getMethod()); $this->assertEquals('201', $metric->getStatus()); + $this->assertEquals('storage', $metric->getService()); $this->assertEquals('bucket', $metric->getResource()); $this->assertEquals('bucket123', $metric->getResourceId()); - $this->assertEquals('US', $metric->getCountry()); - $this->assertEquals('test-agent', $metric->getUserAgent()); - - // Remaining tags should only contain non-event fields - $tags = $metric->getTags(); - $this->assertEquals('us-east', $tags['region'] ?? null); - $this->assertArrayNotHasKey('path', $tags); - $this->assertArrayNotHasKey('method', $tags); - $this->assertArrayNotHasKey('status', $tags); - $this->assertArrayNotHasKey('resource', $tags); - $this->assertArrayNotHasKey('resourceId', $tags); - $this->assertArrayNotHasKey('country', $tags); - $this->assertArrayNotHasKey('userAgent', $tags); + $this->assertEquals('42', $metric->getResourceInternalId()); + $this->assertEquals('team_x', $metric->getTeamId()); + $this->assertEquals('7', $metric->getTeamInternalId()); + // country and region are lowercased on write + $this->assertEquals('us', $metric->getCountry()); + $this->assertEquals('us-east', $metric->getRegion()); + $this->assertEquals('app.example.com', $metric->getHostname()); + $this->assertEquals('IOS', $metric->getOsCode()); + $this->assertEquals('iOS', $metric->getOsName()); + $this->assertEquals('17.4', $metric->getOsVersion()); + $this->assertEquals('mobile-app', $metric->getClientType()); + $this->assertEquals('APW', $metric->getClientCode()); + $this->assertEquals('Appwrite SDK', $metric->getClientName()); + $this->assertEquals('15.0.0', $metric->getClientVersion()); + $this->assertEquals('WebKit', $metric->getClientEngine()); + $this->assertEquals('605', $metric->getClientEngineVersion()); + $this->assertEquals('smartphone', $metric->getDeviceName()); + $this->assertEquals('Apple', $metric->getDeviceBrand()); + $this->assertEquals('iPhone 13', $metric->getDeviceModel()); + } + + /** + * Gauge rows round-trip the four gauge dimension columns. + */ + public function testGaugeColumnsRoundTrip(): void + { + $this->usage->purge([], Usage::TYPE_GAUGE); + + $this->assertTrue($this->usage->addBatch([ + [ + 'metric' => 'gauge-cols-test', + 'value' => 500, + 'tags' => [ + 'teamId' => 'team_x', + 'teamInternalId' => '7', + 'resourceId' => 'r1', + 'resourceInternalId' => '42', + ], + ], + ], Usage::TYPE_GAUGE)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['gauge-cols-test']), + ], Usage::TYPE_GAUGE); + + $this->assertCount(1, $results); + $metric = $results[0]; + $this->assertEquals('team_x', $metric->getTeamId()); + $this->assertEquals('7', $metric->getTeamInternalId()); + $this->assertEquals('r1', $metric->getResourceId()); + $this->assertEquals('42', $metric->getResourceInternalId()); + } + + public function testUnknownTagKeyThrows(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessageMatches("/Unknown column 'bogus'/"); + $this->usage->addBatch([ + ['metric' => 'x', 'value' => 1, 'tags' => ['bogus' => 'v']], + ], Usage::TYPE_EVENT); + } + + public function testCountryLowercased(): void + { + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'lc-country', 'value' => 1, 'tags' => ['country' => 'US']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['lc-country']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertSame('us', $results[0]->getCountry()); + } + + public function testRegionLowercased(): void + { + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'lc-region', 'value' => 1, 'tags' => ['region' => 'FR']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['lc-region']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertSame('fr', $results[0]->getRegion()); + } + + public function testEmptyStringCoercedToNull(): void + { + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'empty-string', 'value' => 1, 'tags' => ['osName' => '']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['empty-string']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertNull($results[0]->getOsName()); + } + + /** + * Round-trip a row that exercises every queryable dimension column to + * confirm the events table schema accepts and persists each one. + */ + public function testEventsSchemaPersistsAllNewColumns(): void + { + $this->usage->purge([], Usage::TYPE_EVENT); + + $tags = [ + 'path' => '/v1/x', 'method' => 'GET', 'status' => '200', + 'service' => 'storage', 'resource' => 'bucket', + 'resourceId' => 'r1', 'resourceInternalId' => '42', + 'teamId' => 't1', 'teamInternalId' => '7', + 'country' => 'us', 'region' => 'fra', 'hostname' => 'h.example.com', + 'osCode' => 'IOS', 'osName' => 'iOS', 'osVersion' => '17.4', + 'clientType' => 'browser', 'clientCode' => 'CH', + 'clientName' => 'Chrome', 'clientVersion' => '125', + 'clientEngine' => 'Blink', 'clientEngineVersion' => '125', + 'deviceName' => 'desktop', 'deviceBrand' => 'Apple', + 'deviceModel' => 'MacBook', + ]; + + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'schema-roundtrip', 'value' => 1, 'tags' => $tags], + ], Usage::TYPE_EVENT)); + + // Filtering on each indexed dimension should be schema-valid. + foreach (['service', 'resourceInternalId', 'teamId', 'teamInternalId', 'region', 'hostname', 'osName', 'clientName', 'deviceName'] as $col) { + $value = $tags[$col]; + $expected = $col === 'region' ? strtolower($value) : $value; + $rows = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['schema-roundtrip']), + \Utopia\Query\Query::equal($col, [$expected]), + ], Usage::TYPE_EVENT); + $this->assertGreaterThanOrEqual(1, count($rows), "Filter on {$col} returned no rows"); + } } /** @@ -317,7 +465,7 @@ public function testGaugeTableSimpleSchema(): void $this->usage->purge([], Usage::TYPE_GAUGE); $this->assertTrue($this->usage->addBatch([ - ['metric' => 'gauge-simple', 'value' => 500, 'tags' => ['region' => 'us-east']], + ['metric' => 'gauge-simple', 'value' => 500, 'tags' => ['resourceId' => 'r1']], ], Usage::TYPE_GAUGE)); $results = $this->usage->find([ @@ -333,7 +481,7 @@ public function testGaugeTableSimpleSchema(): void $this->assertNull($results[0]->getMethod()); $this->assertNull($results[0]->getStatus()); $this->assertNull($results[0]->getResource()); - $this->assertNull($results[0]->getResourceId()); + $this->assertEquals('r1', $results[0]->getResourceId()); } /** @@ -430,7 +578,7 @@ public function testMetricsWithSpecialCharacters(): void { $specialVal = "Text with \n newline, \t tab, \"quote\", and unicode \u{1F600}"; $this->assertTrue($this->usage->addBatch([ - ['metric' => 'special-metric', 'value' => 1, 'tags' => ['s' => $specialVal]], + ['metric' => 'special-metric', 'value' => 1, 'tags' => ['hostname' => $specialVal]], ], Usage::TYPE_EVENT)); $results = $this->usage->find([ @@ -439,8 +587,7 @@ public function testMetricsWithSpecialCharacters(): void $this->assertEquals(1, count($results)); $this->assertEquals('special-metric', $results[0]->getMetric()); - $tags = $results[0]->getTags(); - $this->assertEquals($specialVal, $tags['s']); + $this->assertEquals($specialVal, $results[0]->getHostname()); } /** @@ -453,10 +600,10 @@ public function testFindComprehensive(): void // Setup test data $this->usage->addBatch([ - ['metric' => 'metric-A', 'value' => 10, 'tags' => ['category' => 'cat1']], + ['metric' => 'metric-A', 'value' => 10, 'tags' => ['service' => 'cat1']], ], Usage::TYPE_EVENT); $this->usage->addBatch([ - ['metric' => 'metric-B', 'value' => 20, 'tags' => ['category' => 'cat2']], + ['metric' => 'metric-B', 'value' => 20, 'tags' => ['service' => 'cat2']], ], Usage::TYPE_EVENT); // 1. Array Equal (IN) @@ -707,9 +854,9 @@ public function testCompression(): void // Insert data using addBatch with compression enabled $batchResult = $usage->addBatch([ - ['metric' => 'compression.test.batch', 'value' => 50, 'tags' => ['type' => 'batch']], - ['metric' => 'compression.test.batch', 'value' => 75, 'tags' => ['type' => 'batch']], - ['metric' => 'compression.test.single', 'value' => 100, 'tags' => ['type' => 'single']], + ['metric' => 'compression.test.batch', 'value' => 50, 'tags' => ['service' => 'batch']], + ['metric' => 'compression.test.batch', 'value' => 75, 'tags' => ['service' => 'batch']], + ['metric' => 'compression.test.single', 'value' => 100, 'tags' => ['service' => 'single']], ], Usage::TYPE_EVENT); $this->assertTrue($batchResult); @@ -774,7 +921,7 @@ public function testConnectionPooling(): void // Make some requests $usage->addBatch([ - ['metric' => 'pooling.test', 'value' => 100, 'tags' => ['test' => 'value']], + ['metric' => 'pooling.test', 'value' => 100, 'tags' => ['service' => 'value']], ], Usage::TYPE_EVENT); $usage->find([], Usage::TYPE_EVENT); $usage->count([], Usage::TYPE_EVENT); @@ -885,7 +1032,7 @@ public function testRetryWithSuccessfulOperations(): void // These operations should succeed on first attempt (no retries needed) $result = $usage->addBatch([ - ['metric' => 'retry.test', 'value' => 100, 'tags' => ['test' => 'success']], + ['metric' => 'retry.test', 'value' => 100, 'tags' => ['service' => 'success']], ], Usage::TYPE_EVENT); $this->assertTrue($result); @@ -1094,6 +1241,79 @@ public function testCursorWithGroupByIntervalThrows(): void ], Usage::TYPE_EVENT); } + public function testGroupByServiceDailyAggregates(): void + { + $this->usage->purge(); + + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'gb-service', 'value' => 10, 'tags' => ['service' => 'storage']], + ['metric' => 'gb-service', 'value' => 25, 'tags' => ['service' => 'storage']], + ['metric' => 'gb-service', 'value' => 5, 'tags' => ['service' => 'databases']], + ], Usage::TYPE_EVENT)); + + $start = (new \DateTime())->modify('-1 day')->format('Y-m-d\TH:i:s'); + $end = (new \DateTime())->modify('+1 day')->format('Y-m-d\TH:i:s'); + + $results = $this->usage->find([ + UsageQuery::groupByInterval('time', '1d'), + UsageQuery::groupBy('service'), + Query::equal('metric', ['gb-service']), + Query::greaterThanEqual('time', $start), + Query::lessThanEqual('time', $end), + ], Usage::TYPE_EVENT); + + $this->assertGreaterThanOrEqual(2, count($results)); + + $byService = []; + foreach ($results as $row) { + $service = $row->getService(); + $this->assertNotNull($service, 'groupBy(service) should surface the service dimension on each Metric'); + $byService[$service] = ($byService[$service] ?? 0) + $row->getValue(); + } + + $this->assertEquals(35, $byService['storage'] ?? null); + $this->assertEquals(5, $byService['databases'] ?? null); + } + + public function testGroupByMultipleDimensionsHourly(): void + { + $this->usage->purge(); + + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'gb-multi', 'value' => 1, 'tags' => ['service' => 'storage', 'path' => '/v1/a']], + ['metric' => 'gb-multi', 'value' => 2, 'tags' => ['service' => 'storage', 'path' => '/v1/a']], + ['metric' => 'gb-multi', 'value' => 4, 'tags' => ['service' => 'storage', 'path' => '/v1/b']], + ['metric' => 'gb-multi', 'value' => 8, 'tags' => ['service' => 'databases', 'path' => '/v1/a']], + ], Usage::TYPE_EVENT)); + + $start = (new \DateTime())->modify('-1 hour')->format('Y-m-d\TH:i:s'); + $end = (new \DateTime())->modify('+1 hour')->format('Y-m-d\TH:i:s'); + + $results = $this->usage->find([ + UsageQuery::groupByInterval('time', '1h'), + UsageQuery::groupBy('service'), + UsageQuery::groupBy('path'), + Query::equal('metric', ['gb-multi']), + Query::greaterThanEqual('time', $start), + Query::lessThanEqual('time', $end), + ], Usage::TYPE_EVENT); + + $this->assertGreaterThanOrEqual(3, count($results)); + + $byPair = []; + foreach ($results as $row) { + $svc = $row->getService(); + $path = $row->getPath(); + $this->assertNotNull($svc); + $this->assertNotNull($path); + $byPair["{$svc}|{$path}"] = ($byPair["{$svc}|{$path}"] ?? 0) + $row->getValue(); + } + + $this->assertEquals(3, $byPair['storage|/v1/a'] ?? null); + $this->assertEquals(4, $byPair['storage|/v1/b'] ?? null); + $this->assertEquals(8, $byPair['databases|/v1/a'] ?? null); + } + public function testNotEqualQuery(): void { // Fixture: requests x2, bandwidth x1 in events diff --git a/tests/Usage/Adapter/DatabaseTest.php b/tests/Usage/Adapter/DatabaseTest.php index 5e0c05a..779450c 100644 --- a/tests/Usage/Adapter/DatabaseTest.php +++ b/tests/Usage/Adapter/DatabaseTest.php @@ -21,8 +21,12 @@ class DatabaseTest extends TestCase protected function initializeUsage(): void { - $dbHost = 'mariadb'; - $dbPort = '3306'; + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed in this environment'); + } + + $dbHost = getenv('MARIADB_HOST') ?: 'mariadb'; + $dbPort = getenv('MARIADB_PORT') ?: '3306'; $dbUser = 'root'; $dbPass = 'password'; @@ -50,6 +54,149 @@ protected function initializeUsage(): void } } + /** + * Round-trip a row with the full event dimension set through the + * Database adapter. + */ + public function testEventColumnsExtractedFromTags(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->usage->purge([], Usage::TYPE_EVENT); + + $this->assertTrue($this->usage->addBatch([ + [ + 'metric' => 'event-cols-db', + 'value' => 42, + 'tags' => [ + 'path' => '/v1/storage/files', + 'method' => 'POST', + 'status' => '201', + 'service' => 'storage', + 'resource' => 'bucket', + 'resourceId' => 'bucket123', + 'resourceInternalId' => '42', + 'teamId' => 'team_x', + 'teamInternalId' => '7', + 'country' => 'US', + 'region' => 'us-east', + 'hostname' => 'app.example.com', + 'osName' => 'iOS', + 'clientName' => 'Appwrite SDK', + 'deviceName' => 'smartphone', + ], + ], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['event-cols-db']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $metric = $results[0]; + $this->assertEquals('/v1/storage/files', $metric->getPath()); + $this->assertEquals('storage', $metric->getService()); + $this->assertEquals('42', $metric->getResourceInternalId()); + $this->assertEquals('team_x', $metric->getTeamId()); + $this->assertEquals('7', $metric->getTeamInternalId()); + $this->assertEquals('us', $metric->getCountry()); + $this->assertEquals('us-east', $metric->getRegion()); + $this->assertEquals('app.example.com', $metric->getHostname()); + $this->assertEquals('iOS', $metric->getOsName()); + $this->assertEquals('Appwrite SDK', $metric->getClientName()); + $this->assertEquals('smartphone', $metric->getDeviceName()); + } + + /** + * Gauge rows round-trip the four gauge dimension columns. + */ + public function testGaugeColumnsRoundTrip(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->usage->purge([], Usage::TYPE_GAUGE); + + $this->assertTrue($this->usage->addBatch([ + [ + 'metric' => 'gauge-cols-db', + 'value' => 500, + 'tags' => [ + 'teamId' => 'team_x', + 'teamInternalId' => '7', + 'resourceId' => 'r1', + 'resourceInternalId' => '42', + ], + ], + ], Usage::TYPE_GAUGE)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['gauge-cols-db']), + ], Usage::TYPE_GAUGE); + + $this->assertCount(1, $results); + $metric = $results[0]; + $this->assertEquals('team_x', $metric->getTeamId()); + $this->assertEquals('7', $metric->getTeamInternalId()); + $this->assertEquals('r1', $metric->getResourceId()); + $this->assertEquals('42', $metric->getResourceInternalId()); + } + + public function testUnknownTagKeyThrows(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->expectException(\Exception::class); + $this->expectExceptionMessageMatches("/Unknown column 'bogus'/"); + $this->usage->addBatch([ + ['metric' => 'x', 'value' => 1, 'tags' => ['bogus' => 'v']], + ], Usage::TYPE_EVENT); + } + + public function testCountryAndRegionLowercased(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'lc-db', 'value' => 1, 'tags' => ['country' => 'US', 'region' => 'FR']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['lc-db']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertSame('us', $results[0]->getCountry()); + $this->assertSame('fr', $results[0]->getRegion()); + } + + public function testEmptyStringCoercedToNull(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'empty-db', 'value' => 1, 'tags' => ['osName' => '']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['empty-db']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertNull($results[0]->getOsName()); + } + /** * Test healthCheck() method */ @@ -78,9 +225,13 @@ public function testHealthCheck(): void */ public function testHealthCheckWithNonExistentDatabase(): void { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + // Create a new database instance pointing to a non-existent database - $dbHost = 'mariadb'; - $dbPort = '3306'; + $dbHost = getenv('MARIADB_HOST') ?: 'mariadb'; + $dbPort = getenv('MARIADB_PORT') ?: '3306'; $dbUser = 'root'; $dbPass = 'password'; diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index 5a05c07..955379e 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -15,63 +15,29 @@ public function testGetEventSchemaReturnsAttributeDefinitions(): void $schema = Metric::getEventSchema(); $this->assertIsArray($schema); - $this->assertCount(11, $schema); + // 3 core (metric, value, time) + 24 dimension columns from EVENT_COLUMNS. + $this->assertCount(3 + count(Metric::EVENT_COLUMNS), $schema); - // Test metric attribute $metricAttr = $schema[0]; $this->assertEquals('metric', $metricAttr['$id']); $this->assertEquals('string', $metricAttr['type']); $this->assertEquals(255, $metricAttr['size']); $this->assertTrue($metricAttr['required']); - // Test value attribute $valueAttr = $schema[1]; $this->assertEquals('value', $valueAttr['$id']); $this->assertEquals('integer', $valueAttr['type']); $this->assertTrue($valueAttr['required']); - // Test time attribute (optional) $timeAttr = $schema[2]; $this->assertEquals('time', $timeAttr['$id']); $this->assertEquals('datetime', $timeAttr['type']); $this->assertFalse($timeAttr['required']); - // Test event-specific columns - $pathAttr = $schema[3]; - $this->assertEquals('path', $pathAttr['$id']); - $this->assertFalse($pathAttr['required']); - - $methodAttr = $schema[4]; - $this->assertEquals('method', $methodAttr['$id']); - $this->assertFalse($methodAttr['required']); - - $statusAttr = $schema[5]; - $this->assertEquals('status', $statusAttr['$id']); - $this->assertFalse($statusAttr['required']); - - $resourceAttr = $schema[6]; - $this->assertEquals('resource', $resourceAttr['$id']); - $this->assertFalse($resourceAttr['required']); - - $resourceIdAttr = $schema[7]; - $this->assertEquals('resourceId', $resourceIdAttr['$id']); - $this->assertFalse($resourceIdAttr['required']); - - // Test country attribute (optional) - $countryAttr = $schema[8]; - $this->assertEquals('country', $countryAttr['$id']); - $this->assertFalse($countryAttr['required']); - - // Test userAgent attribute (optional) - $userAgentAttr = $schema[9]; - $this->assertEquals('userAgent', $userAgentAttr['$id']); - $this->assertFalse($userAgentAttr['required']); - - // Test tags attribute (optional) - $tagsAttr = $schema[10]; - $this->assertEquals('tags', $tagsAttr['$id']); - $this->assertEquals('string', $tagsAttr['type']); - $this->assertFalse($tagsAttr['required']); + $ids = array_column($schema, '$id'); + foreach (Metric::EVENT_COLUMNS as $col) { + $this->assertContains($col, $ids, "Event schema missing dimension column {$col}"); + } } /** @@ -82,12 +48,17 @@ public function testGetGaugeSchemaReturnsAttributeDefinitions(): void $schema = Metric::getGaugeSchema(); $this->assertIsArray($schema); - $this->assertCount(4, $schema); + // 3 core (metric, value, time) + 4 GAUGE_COLUMNS. + $this->assertCount(3 + count(Metric::GAUGE_COLUMNS), $schema); $this->assertEquals('metric', $schema[0]['$id']); $this->assertEquals('value', $schema[1]['$id']); $this->assertEquals('time', $schema[2]['$id']); - $this->assertEquals('tags', $schema[3]['$id']); + + $ids = array_column($schema, '$id'); + foreach (Metric::GAUGE_COLUMNS as $col) { + $this->assertContains($col, $ids); + } } /** @@ -101,39 +72,58 @@ public function testGetSchemaReturnsEventSchema(): void } /** - * Test Metric::getEventIndexes() returns correct index definitions + * Test Metric::getEventIndexes() returns one entry per indexed dimension. */ public function testGetEventIndexesReturnsIndexDefinitions(): void { $indexes = Metric::getEventIndexes(); $this->assertIsArray($indexes); - // metric/time are now covered by the primary key (ORDER BY (tenant, - // metric, time, id)), so only event-specific secondary indexes - // remain: path, method, status, resource, resourceId, country, - // userAgent. - $this->assertCount(7, $indexes); - $this->assertEquals('index-path', $indexes[0]['$id']); - $this->assertEquals('index-method', $indexes[1]['$id']); - $this->assertEquals('index-status', $indexes[2]['$id']); - $this->assertEquals('index-resource', $indexes[3]['$id']); - $this->assertEquals('index-resourceId', $indexes[4]['$id']); - $this->assertEquals('index-country', $indexes[5]['$id']); - $this->assertEquals('index-userAgent', $indexes[6]['$id']); + $ids = array_column($indexes, '$id'); + $this->assertNotContains('index-userAgent', $ids, 'userAgent index must be dropped'); } /** - * Test Metric::getGaugeIndexes() returns correct index definitions + * Test Metric::getGaugeIndexes() returns one entry per gauge dimension. */ public function testGetGaugeIndexesReturnsIndexDefinitions(): void { $indexes = Metric::getGaugeIndexes(); - // Gauges only filter by metric and time, both in the primary key, - // so no secondary indexes are needed. $this->assertIsArray($indexes); - $this->assertCount(0, $indexes); + $this->assertCount(count(Metric::GAUGE_COLUMNS), $indexes); + } + + public function testEventIndexesCoverNewFilterableColumns(): void + { + $indexed = []; + foreach (Metric::getEventIndexes() as $idx) { + /** @var array $attrs */ + $attrs = $idx['attributes']; + $indexed = array_merge($indexed, $attrs); + } + foreach ([ + 'path', 'method', 'status', 'service', 'resource', + 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', + 'country', 'region', 'hostname', + 'osName', 'clientType', 'clientName', 'deviceName', + ] as $col) { + $this->assertContains($col, $indexed, "Event indexes missing {$col}"); + } + } + + public function testGaugeIndexesCoverIdColumns(): void + { + $indexed = []; + foreach (Metric::getGaugeIndexes() as $idx) { + /** @var array $attrs */ + $attrs = $idx['attributes']; + $indexed = array_merge($indexed, $attrs); + } + foreach (['resourceId', 'resourceInternalId', 'teamId', 'teamInternalId'] as $col) { + $this->assertContains($col, $indexed); + } } /** @@ -160,7 +150,7 @@ public function testValidateAcceptsValidEventData(): void 'status' => '201', 'resource' => 'bucket', 'resourceId' => 'abc123', - 'tags' => ['region' => 'us-east', 'env' => 'prod'], + 'region' => 'us', ]; // Should not throw exception @@ -177,7 +167,8 @@ public function testValidateAcceptsValidGaugeData(): void 'metric' => 'storage', 'value' => 10000, 'time' => '2024-01-01 12:00:00', - 'tags' => ['region' => 'us-east'], + 'teamId' => 'team1', + 'resourceId' => 'r1', ]; Metric::validate($validData, 'gauge'); @@ -311,36 +302,6 @@ public function testValidateRejectsInvalidDatetimeString(): void ], 'event'); } - /** - * Test Metric::validate() rejects non-array tags - */ - public function testValidateRejectsNonArrayTags(): void - { - $this->expectException(\Exception::class); - $this->expectExceptionMessage("Attribute 'tags' must be an array"); - - Metric::validate([ - 'metric' => 'requests', - 'value' => 100, - 'tags' => 'not-an-array', - ], 'event'); - } - - /** - * Test Metric::validate() accepts empty tags array - */ - public function testValidateAcceptsEmptyTags(): void - { - $data = [ - 'metric' => 'requests', - 'value' => 100, - 'tags' => [], - ]; - - Metric::validate($data, 'event'); - $this->assertTrue(true); - } - /** * Test Metric constructor initializes with data */ @@ -356,7 +317,6 @@ public function testConstructorInitializesWithData(): void 'status' => '201', 'resource' => 'bucket', 'resourceId' => 'abc123', - 'tags' => ['env' => 'prod'], ]; $metric = new Metric($data); @@ -370,7 +330,6 @@ public function testConstructorInitializesWithData(): void $this->assertEquals('201', $metric->getStatus()); $this->assertEquals('bucket', $metric->getResource()); $this->assertEquals('abc123', $metric->getResourceId()); - $this->assertEquals(['env' => 'prod'], $metric->getTags()); } /** @@ -487,25 +446,6 @@ public function testGetTimeReturnsNullWhenNotSet(): void $this->assertNull($metric->getTime()); } - /** - * Test Metric::getTags() returns tags - */ - public function testGetTagsReturnsTags(): void - { - $tags = ['region' => 'us-east', 'env' => 'prod']; - $metric = new Metric(['tags' => $tags]); - $this->assertEquals($tags, $metric->getTags()); - } - - /** - * Test Metric::getTags() returns empty array when not set - */ - public function testGetTagsReturnsEmptyArrayWhenNotSet(): void - { - $metric = new Metric([]); - $this->assertEquals([], $metric->getTags()); - } - /** * Test Metric::getTenant() returns tenant ID as string */ @@ -671,7 +611,96 @@ public function testToArrayReturnsArray(): void */ public function testEventColumnsConstant(): void { - $expected = ['path', 'method', 'status', 'resource', 'resourceId', 'country', 'userAgent']; - $this->assertEquals($expected, Metric::EVENT_COLUMNS); + $expected = [ + 'path', 'method', 'status', + 'service', 'resource', 'resourceId', 'resourceInternalId', + 'teamId', 'teamInternalId', + 'country', 'region', 'hostname', + 'osCode', 'osName', 'osVersion', + 'clientType', 'clientCode', 'clientName', 'clientVersion', + 'clientEngine', 'clientEngineVersion', + 'deviceName', 'deviceBrand', 'deviceModel', + ]; + $this->assertSame($expected, Metric::EVENT_COLUMNS); + } + + /** + * Test GAUGE_COLUMNS constant + */ + public function testGaugeColumnsConstant(): void + { + $expected = ['teamId', 'teamInternalId', 'resourceId', 'resourceInternalId']; + $this->assertSame($expected, Metric::GAUGE_COLUMNS); + } + + /** + * Test that the event schema contains every new dimension column. + */ + public function testEventSchemaHasAllNewColumns(): void + { + $ids = array_column(Metric::getEventSchema(), '$id'); + foreach ([ + 'service', 'resourceInternalId', 'teamId', 'teamInternalId', + 'region', 'hostname', 'osCode', 'osName', 'osVersion', + 'clientType', 'clientCode', 'clientName', 'clientVersion', + 'clientEngine', 'clientEngineVersion', + 'deviceName', 'deviceBrand', 'deviceModel', + ] as $col) { + $this->assertContains($col, $ids, "Event schema missing {$col}"); + } + $this->assertNotContains('userAgent', $ids, 'userAgent must be removed'); + $this->assertNotContains('tags', $ids, 'tags must be removed'); + } + + /** + * Test that the gauge schema contains the new team and resource id columns. + */ + public function testGaugeSchemaHasTeamAndResourceColumns(): void + { + $ids = array_column(Metric::getGaugeSchema(), '$id'); + foreach (['teamId', 'teamInternalId', 'resourceId', 'resourceInternalId'] as $col) { + $this->assertContains($col, $ids); + } + $this->assertNotContains('tags', $ids); + } + + /** + * Test the 18 new typed getters return the value passed via the constructor. + */ + public function testNewGettersReturnString(): void + { + $m = new Metric([ + 'service' => 'storage', 'resourceInternalId' => '42', + 'teamId' => 'org_x', 'teamInternalId' => '7', + 'region' => 'fra', 'hostname' => 'app.example.com', + 'osCode' => 'IOS', 'osName' => 'iOS', 'osVersion' => '17.4', + 'clientType' => 'mobile-app', 'clientCode' => 'APW', 'clientName' => 'Appwrite SDK', + 'clientVersion' => '15.0.0', 'clientEngine' => 'WebKit', 'clientEngineVersion' => '605', + 'deviceName' => 'smartphone', 'deviceBrand' => 'Apple', 'deviceModel' => 'iPhone 13', + ]); + $this->assertSame('storage', $m->getService()); + $this->assertSame('42', $m->getResourceInternalId()); + $this->assertSame('org_x', $m->getTeamId()); + $this->assertSame('7', $m->getTeamInternalId()); + $this->assertSame('fra', $m->getRegion()); + $this->assertSame('app.example.com', $m->getHostname()); + $this->assertSame('IOS', $m->getOsCode()); + $this->assertSame('iOS', $m->getOsName()); + $this->assertSame('17.4', $m->getOsVersion()); + $this->assertSame('mobile-app', $m->getClientType()); + $this->assertSame('APW', $m->getClientCode()); + $this->assertSame('Appwrite SDK', $m->getClientName()); + $this->assertSame('15.0.0', $m->getClientVersion()); + $this->assertSame('WebKit', $m->getClientEngine()); + $this->assertSame('605', $m->getClientEngineVersion()); + $this->assertSame('smartphone', $m->getDeviceName()); + $this->assertSame('Apple', $m->getDeviceBrand()); + $this->assertSame('iPhone 13', $m->getDeviceModel()); + } + + public function testDroppedGettersDoNotExist(): void + { + $this->assertFalse(method_exists(Metric::class, 'getUserAgent')); + $this->assertFalse(method_exists(Metric::class, 'getTags')); } } diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index 6bb71d0..6ac3a37 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -34,7 +34,7 @@ public function createUsageMetrics(): void // Gauges: point-in-time snapshots $this->assertTrue($this->usage->addBatch([ - ['metric' => 'storage', 'value' => 10000, 'tags' => ['region' => 'us-east']], + ['metric' => 'storage', 'value' => 10000, 'tags' => ['resourceId' => 'p1']], ], Usage::TYPE_GAUGE)); } @@ -639,4 +639,27 @@ public function testGroupByIntervalWithLimitOffset(): void $this->assertGreaterThanOrEqual(1, count($results)); } + + public function testGroupByUnknownAttributeThrows(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessageMatches("/groupBy attribute 'not_a_column'/"); + + $this->usage->find([ + UsageQuery::groupByInterval('time', '1h'), + UsageQuery::groupBy('not_a_column'), + Query::equal('metric', ['gbi-requests']), + ], Usage::TYPE_EVENT); + } + + public function testGroupByWithoutGroupByIntervalThrows(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessageMatches('/groupBy requires groupByInterval/'); + + $this->usage->find([ + UsageQuery::groupBy('service'), + Query::equal('metric', ['gbi-requests']), + ], Usage::TYPE_EVENT); + } } diff --git a/tests/Usage/UsageQueryTest.php b/tests/Usage/UsageQueryTest.php index b4b07b1..31da415 100644 --- a/tests/Usage/UsageQueryTest.php +++ b/tests/Usage/UsageQueryTest.php @@ -131,4 +131,104 @@ public function testUsageQueryExtendsQuery(): void $query = UsageQuery::groupByInterval('time', '1h'); $this->assertInstanceOf(Query::class, $query); } + + public function testGroupByCreation(): void + { + $query = UsageQuery::groupBy('service'); + + $this->assertInstanceOf(UsageQuery::class, $query); + $this->assertEquals(UsageQuery::TYPE_GROUP_BY, $query->getMethod()); + $this->assertEquals('service', $query->getAttribute()); + $this->assertEquals([], $query->getValues()); + } + + public function testGroupByIsMethod(): void + { + $this->assertTrue(UsageQuery::isMethod(UsageQuery::TYPE_GROUP_BY)); + $this->assertTrue(UsageQuery::isMethod(UsageQuery::TYPE_GROUP_BY_INTERVAL)); + $this->assertTrue(UsageQuery::isMethod(Query::TYPE_EQUAL)); + $this->assertFalse(UsageQuery::isMethod('notARealMethod')); + } + + public function testIsGroupBy(): void + { + $groupBy = UsageQuery::groupBy('service'); + $groupByInterval = UsageQuery::groupByInterval('time', '1h'); + $regular = Query::equal('metric', ['bandwidth']); + + $this->assertTrue(UsageQuery::isGroupBy($groupBy)); + $this->assertFalse(UsageQuery::isGroupBy($groupByInterval)); + $this->assertFalse(UsageQuery::isGroupBy($regular)); + } + + public function testExtractGroupByReturnsAllMatches(): void + { + $byService = UsageQuery::groupBy('service'); + $byPath = UsageQuery::groupBy('path'); + $interval = UsageQuery::groupByInterval('time', '1h'); + $equal = Query::equal('metric', ['bandwidth']); + + $queries = [$equal, $byService, $interval, $byPath]; + + $extracted = UsageQuery::extractGroupBy($queries); + + $this->assertCount(2, $extracted); + $this->assertEquals('service', $extracted[0]->getAttribute()); + $this->assertEquals('path', $extracted[1]->getAttribute()); + } + + public function testExtractGroupByReturnsEmptyWhenAbsent(): void + { + $queries = [ + Query::equal('metric', ['bandwidth']), + UsageQuery::groupByInterval('time', '1h'), + ]; + + $this->assertSame([], UsageQuery::extractGroupBy($queries)); + } + + public function testRemoveGroupBy(): void + { + $byService = UsageQuery::groupBy('service'); + $byPath = UsageQuery::groupBy('path'); + $interval = UsageQuery::groupByInterval('time', '1h'); + $equal = Query::equal('metric', ['bandwidth']); + + $queries = [$equal, $byService, $interval, $byPath]; + + $remaining = UsageQuery::removeGroupBy($queries); + + $this->assertCount(2, $remaining); + foreach ($remaining as $query) { + $this->assertNotEquals(UsageQuery::TYPE_GROUP_BY, $query->getMethod()); + } + } + + public function testGroupByParseRoundTrip(): void + { + $json = json_encode([ + 'method' => UsageQuery::TYPE_GROUP_BY, + 'attribute' => 'service', + 'values' => [], + ]); + $this->assertIsString($json); + + $parsed = UsageQuery::parse($json); + + $this->assertEquals(UsageQuery::TYPE_GROUP_BY, $parsed->getMethod()); + $this->assertEquals('service', $parsed->getAttribute()); + $this->assertEquals([], $parsed->getValues()); + } + + public function testExtractGroupByFromParsedQuery(): void + { + // Queries created via Query::parse() are base Query objects, not UsageQuery. + $parsedGroupBy = new Query(UsageQuery::TYPE_GROUP_BY, 'service', []); + $equal = Query::equal('metric', ['bandwidth']); + + $extracted = UsageQuery::extractGroupBy([$equal, $parsedGroupBy]); + + $this->assertCount(1, $extracted); + $this->assertEquals('service', $extracted[0]->getAttribute()); + } }