From 978c52406f385e64aadfdf306cb5f0d3ab715434 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:28:50 +0000 Subject: [PATCH 01/18] feat(metric): widen EVENT_COLUMNS, add GAUGE_COLUMNS --- src/Usage/Metric.php | 16 +++++++++++++++- tests/Usage/MetricTest.php | 22 ++++++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index 3f26450..11e56ad 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -40,7 +40,21 @@ class Metric extends ArrayObject /** * Event-specific column names that are extracted from tags into dedicated columns. */ - public const EVENT_COLUMNS = ['path', 'method', 'status', 'resource', 'resourceId', 'country', 'userAgent']; + public const EVENT_COLUMNS = [ + 'path', 'method', 'status', + 'service', 'resource', 'resourceId', 'resourceInternalId', + 'teamId', 'teamInternalId', + 'country', 'region', 'hostname', + 'osCode', 'osName', 'osVersion', + 'clientType', 'clientCode', 'clientName', 'clientVersion', + 'clientEngine', 'clientEngineVersion', + 'deviceName', 'deviceBrand', 'deviceModel', + ]; + + /** + * Gauge-specific column names that are extracted from tags into dedicated columns. + */ + public const GAUGE_COLUMNS = ['teamId', 'teamInternalId', 'resourceId', 'resourceInternalId']; /** * Construct a new metric object. diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index 5a05c07..2c901a8 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -671,7 +671,25 @@ public function testToArrayReturnsArray(): void */ public function testEventColumnsConstant(): void { - $expected = ['path', 'method', 'status', 'resource', 'resourceId', 'country', 'userAgent']; - $this->assertEquals($expected, Metric::EVENT_COLUMNS); + $expected = [ + 'path', 'method', 'status', + 'service', 'resource', 'resourceId', 'resourceInternalId', + 'teamId', 'teamInternalId', + 'country', 'region', 'hostname', + 'osCode', 'osName', 'osVersion', + 'clientType', 'clientCode', 'clientName', 'clientVersion', + 'clientEngine', 'clientEngineVersion', + 'deviceName', 'deviceBrand', 'deviceModel', + ]; + $this->assertSame($expected, Metric::EVENT_COLUMNS); + } + + /** + * Test GAUGE_COLUMNS constant + */ + public function testGaugeColumnsConstant(): void + { + $expected = ['teamId', 'teamInternalId', 'resourceId', 'resourceInternalId']; + $this->assertSame($expected, Metric::GAUGE_COLUMNS); } } From 0002ddd9706aab418a8d08cfa933a5dc3c7ddc94 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:30:01 +0000 Subject: [PATCH 02/18] feat(metric): regenerate event/gauge schemas to match new column set --- .phpunit.result.cache | 2 +- src/Usage/Metric.php | 129 ++++++++++++++----------------------- tests/Usage/MetricTest.php | 86 +++++++++++++------------ 3 files changed, 93 insertions(+), 124 deletions(-) diff --git a/.phpunit.result.cache b/.phpunit.result.cache index a99474f..8ca16f4 100644 --- a/.phpunit.result.cache +++ b/.phpunit.result.cache @@ -1 +1 @@ -{"version":1,"defects":{"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":3,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":3},"times":{"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalCreation":0.001,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalAllValidIntervals":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidIntervalEmpty":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testIsGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByIntervalReturnsNullWhenMissing":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testRemoveGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testValidIntervalsConstant":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testUsageQueryExtendsQuery":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventSchemaReturnsAttributeDefinitions":0.001,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeSchemaReturnsAttributeDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetSchemaReturnsEventSchema":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIndexesReturnsEventIndexes":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidEventData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidGaugeData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsMinimalData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonStringMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsOversizedMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonIntegerValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDateTimeForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDatetimeStringForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsInvalidDatetimeString":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonArrayTags":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsEmptyTags":0,"Utopia\\Tests\\Usage\\MetricTest::testConstructorInitializesWithData":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsMetricId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsEmptyStringWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetMetricReturnsMetricName":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsType":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsDefaultType":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnCorrectValues":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsTimestamp":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsTags":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsEmptyArrayWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsTenantId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantConvertsNumericToString":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributesReturnsAllAttributes":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSetsAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSupportsChaining":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsTrueWhenExists":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsFalseWhenNotExists":0,"Utopia\\Tests\\Usage\\MetricTest::testRemoveAttributeRemovesAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsFalseWhenIdSet":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsTrueWhenNoId":0,"Utopia\\Tests\\Usage\\MetricTest::testToArrayReturnsArray":0,"Utopia\\Tests\\Usage\\MetricTest::testEventColumnsConstant":0}} \ No newline at end of file +{"version":1,"defects":{"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":3,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":3,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonArrayTags":3},"times":{"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalCreation":0.001,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalAllValidIntervals":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidIntervalEmpty":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testIsGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByIntervalReturnsNullWhenMissing":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testRemoveGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testValidIntervalsConstant":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testUsageQueryExtendsQuery":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventSchemaReturnsAttributeDefinitions":0.001,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeSchemaReturnsAttributeDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetSchemaReturnsEventSchema":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIndexesReturnsEventIndexes":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidEventData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidGaugeData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsMinimalData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonStringMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsOversizedMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonIntegerValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDateTimeForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDatetimeStringForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsInvalidDatetimeString":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonArrayTags":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsEmptyTags":0,"Utopia\\Tests\\Usage\\MetricTest::testConstructorInitializesWithData":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsMetricId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsEmptyStringWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetMetricReturnsMetricName":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsType":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsDefaultType":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnCorrectValues":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsTimestamp":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsTags":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsEmptyArrayWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsTenantId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantConvertsNumericToString":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributesReturnsAllAttributes":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSetsAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSupportsChaining":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsTrueWhenExists":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsFalseWhenNotExists":0,"Utopia\\Tests\\Usage\\MetricTest::testRemoveAttributeRemovesAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsFalseWhenIdSet":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsTrueWhenNoId":0,"Utopia\\Tests\\Usage\\MetricTest::testToArrayReturnsArray":0,"Utopia\\Tests\\Usage\\MetricTest::testEventColumnsConstant":0,"Utopia\\Tests\\Usage\\MetricTest::testGaugeColumnsConstant":0,"Utopia\\Tests\\Usage\\MetricTest::testEventSchemaHasAllNewColumns":0,"Utopia\\Tests\\Usage\\MetricTest::testGaugeSchemaHasTeamAndResourceColumns":0}} \ No newline at end of file diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index 11e56ad..e438010 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -422,6 +422,16 @@ public function toArray(): array */ public static function getEventSchema(): array { + $stringColumn = static fn (string $id, int $size): array => [ + '$id' => $id, + 'type' => 'string', + 'size' => $size, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ]; + return [ [ '$id' => 'metric', @@ -451,78 +461,30 @@ public static function getEventSchema(): array 'array' => false, 'filters' => ['datetime'], ], - [ - '$id' => 'path', - 'type' => 'string', - 'size' => 255, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'method', - 'type' => 'string', - 'size' => 16, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'status', - 'type' => 'string', - 'size' => 16, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'resource', - 'type' => 'string', - 'size' => 255, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'resourceId', - 'type' => 'string', - 'size' => 255, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'country', - 'type' => 'string', - 'size' => 2, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'userAgent', - 'type' => 'string', - 'size' => 255, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'tags', - 'type' => 'string', - 'size' => 16777216, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => ['json'], - ], + $stringColumn('path', 1024), + $stringColumn('method', 16), + $stringColumn('status', 16), + $stringColumn('service', 256), + $stringColumn('resource', 256), + $stringColumn('resourceId', 255), + $stringColumn('resourceInternalId', 255), + $stringColumn('teamId', 255), + $stringColumn('teamInternalId', 255), + $stringColumn('country', 2), + $stringColumn('region', 2), + $stringColumn('hostname', 1024), + $stringColumn('osCode', 256), + $stringColumn('osName', 256), + $stringColumn('osVersion', 255), + $stringColumn('clientType', 256), + $stringColumn('clientCode', 256), + $stringColumn('clientName', 256), + $stringColumn('clientVersion', 255), + $stringColumn('clientEngine', 256), + $stringColumn('clientEngineVersion', 255), + $stringColumn('deviceName', 256), + $stringColumn('deviceBrand', 256), + $stringColumn('deviceModel', 255), ]; } @@ -536,6 +498,16 @@ public static function getEventSchema(): array */ public static function getGaugeSchema(): array { + $stringColumn = static fn (string $id, int $size): array => [ + '$id' => $id, + 'type' => 'string', + 'size' => $size, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ]; + return [ [ '$id' => 'metric', @@ -565,15 +537,10 @@ public static function getGaugeSchema(): array 'array' => false, 'filters' => ['datetime'], ], - [ - '$id' => 'tags', - 'type' => 'string', - 'size' => 16777216, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => ['json'], - ], + $stringColumn('teamId', 255), + $stringColumn('teamInternalId', 255), + $stringColumn('resourceId', 255), + $stringColumn('resourceInternalId', 255), ]; } diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index 2c901a8..9ef9995 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -15,63 +15,29 @@ public function testGetEventSchemaReturnsAttributeDefinitions(): void $schema = Metric::getEventSchema(); $this->assertIsArray($schema); - $this->assertCount(11, $schema); + // 3 core (metric, value, time) + 24 dimension columns from EVENT_COLUMNS. + $this->assertCount(3 + count(Metric::EVENT_COLUMNS), $schema); - // Test metric attribute $metricAttr = $schema[0]; $this->assertEquals('metric', $metricAttr['$id']); $this->assertEquals('string', $metricAttr['type']); $this->assertEquals(255, $metricAttr['size']); $this->assertTrue($metricAttr['required']); - // Test value attribute $valueAttr = $schema[1]; $this->assertEquals('value', $valueAttr['$id']); $this->assertEquals('integer', $valueAttr['type']); $this->assertTrue($valueAttr['required']); - // Test time attribute (optional) $timeAttr = $schema[2]; $this->assertEquals('time', $timeAttr['$id']); $this->assertEquals('datetime', $timeAttr['type']); $this->assertFalse($timeAttr['required']); - // Test event-specific columns - $pathAttr = $schema[3]; - $this->assertEquals('path', $pathAttr['$id']); - $this->assertFalse($pathAttr['required']); - - $methodAttr = $schema[4]; - $this->assertEquals('method', $methodAttr['$id']); - $this->assertFalse($methodAttr['required']); - - $statusAttr = $schema[5]; - $this->assertEquals('status', $statusAttr['$id']); - $this->assertFalse($statusAttr['required']); - - $resourceAttr = $schema[6]; - $this->assertEquals('resource', $resourceAttr['$id']); - $this->assertFalse($resourceAttr['required']); - - $resourceIdAttr = $schema[7]; - $this->assertEquals('resourceId', $resourceIdAttr['$id']); - $this->assertFalse($resourceIdAttr['required']); - - // Test country attribute (optional) - $countryAttr = $schema[8]; - $this->assertEquals('country', $countryAttr['$id']); - $this->assertFalse($countryAttr['required']); - - // Test userAgent attribute (optional) - $userAgentAttr = $schema[9]; - $this->assertEquals('userAgent', $userAgentAttr['$id']); - $this->assertFalse($userAgentAttr['required']); - - // Test tags attribute (optional) - $tagsAttr = $schema[10]; - $this->assertEquals('tags', $tagsAttr['$id']); - $this->assertEquals('string', $tagsAttr['type']); - $this->assertFalse($tagsAttr['required']); + $ids = array_column($schema, '$id'); + foreach (Metric::EVENT_COLUMNS as $col) { + $this->assertContains($col, $ids, "Event schema missing dimension column {$col}"); + } } /** @@ -82,12 +48,17 @@ public function testGetGaugeSchemaReturnsAttributeDefinitions(): void $schema = Metric::getGaugeSchema(); $this->assertIsArray($schema); - $this->assertCount(4, $schema); + // 3 core (metric, value, time) + 4 GAUGE_COLUMNS. + $this->assertCount(3 + count(Metric::GAUGE_COLUMNS), $schema); $this->assertEquals('metric', $schema[0]['$id']); $this->assertEquals('value', $schema[1]['$id']); $this->assertEquals('time', $schema[2]['$id']); - $this->assertEquals('tags', $schema[3]['$id']); + + $ids = array_column($schema, '$id'); + foreach (Metric::GAUGE_COLUMNS as $col) { + $this->assertContains($col, $ids); + } } /** @@ -692,4 +663,35 @@ public function testGaugeColumnsConstant(): void $expected = ['teamId', 'teamInternalId', 'resourceId', 'resourceInternalId']; $this->assertSame($expected, Metric::GAUGE_COLUMNS); } + + /** + * Test that the event schema contains every new dimension column. + */ + public function testEventSchemaHasAllNewColumns(): void + { + $ids = array_column(Metric::getEventSchema(), '$id'); + foreach ([ + 'service', 'resourceInternalId', 'teamId', 'teamInternalId', + 'region', 'hostname', 'osCode', 'osName', 'osVersion', + 'clientType', 'clientCode', 'clientName', 'clientVersion', + 'clientEngine', 'clientEngineVersion', + 'deviceName', 'deviceBrand', 'deviceModel', + ] as $col) { + $this->assertContains($col, $ids, "Event schema missing {$col}"); + } + $this->assertNotContains('userAgent', $ids, 'userAgent must be removed'); + $this->assertNotContains('tags', $ids, 'tags must be removed'); + } + + /** + * Test that the gauge schema contains the new team and resource id columns. + */ + public function testGaugeSchemaHasTeamAndResourceColumns(): void + { + $ids = array_column(Metric::getGaugeSchema(), '$id'); + foreach (['teamId', 'teamInternalId', 'resourceId', 'resourceInternalId'] as $col) { + $this->assertContains($col, $ids); + } + $this->assertNotContains('tags', $ids); + } } From 5c9d7e091a528d77fe5e2ac1d7e86539acdef603 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:31:35 +0000 Subject: [PATCH 03/18] feat(metric): add 18 new typed getters, drop getUserAgent/getTags --- src/Usage/Metric.php | 180 +++++++++++++++++++++++++++++++------ tests/Usage/MetricTest.php | 61 ++++++++----- 2 files changed, 194 insertions(+), 47 deletions(-) diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index e438010..90772f8 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -231,41 +231,169 @@ public function getCountry(): ?string } /** - * Get user agent (event metrics only). + * Get service (event metrics only). * - * @return string|null The user agent string, or null if not set + * @return string|null */ - public function getUserAgent(): ?string + public function getService(): ?string { - $userAgent = $this->getAttribute('userAgent', null); - return is_string($userAgent) ? $userAgent : null; + $v = $this->getAttribute('service', null); + return is_string($v) ? $v : null; } /** - * Get tags. + * Get internal resource id (event/gauge metrics). * - * Returns additional metadata associated with this metric as key-value pairs. - * Tags are useful for filtering, grouping, and contextualizing metrics. - * - * Common tag examples: - * - region: Geographic region (us-east, eu-west) - * - userAgent: Client user agent - * - country: Country code - * - * Note: For event metrics, path/method/status/resource/resourceId are stored - * as dedicated columns, not in tags. Remaining metadata (region, userAgent, etc.) - * stays in the tags JSON. - * - * @return array Associative array of tags + * @return string|null + */ + public function getResourceInternalId(): ?string + { + $v = $this->getAttribute('resourceInternalId', null); + return is_string($v) ? $v : null; + } + + /** + * Get team id (event/gauge metrics). + */ + public function getTeamId(): ?string + { + $v = $this->getAttribute('teamId', null); + return is_string($v) ? $v : null; + } + + /** + * Get team internal id (event/gauge metrics). + */ + public function getTeamInternalId(): ?string + { + $v = $this->getAttribute('teamInternalId', null); + return is_string($v) ? $v : null; + } + + /** + * Get region (event metrics). + */ + public function getRegion(): ?string + { + $v = $this->getAttribute('region', null); + return is_string($v) ? $v : null; + } + + /** + * Get caller hostname (event metrics). + */ + public function getHostname(): ?string + { + $v = $this->getAttribute('hostname', null); + return is_string($v) ? $v : null; + } + + /** + * Get OS short code (event metrics). + */ + public function getOsCode(): ?string + { + $v = $this->getAttribute('osCode', null); + return is_string($v) ? $v : null; + } + + /** + * Get OS name (event metrics). + */ + public function getOsName(): ?string + { + $v = $this->getAttribute('osName', null); + return is_string($v) ? $v : null; + } + + /** + * Get OS version (event metrics). + */ + public function getOsVersion(): ?string + { + $v = $this->getAttribute('osVersion', null); + return is_string($v) ? $v : null; + } + + /** + * Get client type (event metrics). + */ + public function getClientType(): ?string + { + $v = $this->getAttribute('clientType', null); + return is_string($v) ? $v : null; + } + + /** + * Get client short code (event metrics). + */ + public function getClientCode(): ?string + { + $v = $this->getAttribute('clientCode', null); + return is_string($v) ? $v : null; + } + + /** + * Get client name (event metrics). + */ + public function getClientName(): ?string + { + $v = $this->getAttribute('clientName', null); + return is_string($v) ? $v : null; + } + + /** + * Get client version (event metrics). + */ + public function getClientVersion(): ?string + { + $v = $this->getAttribute('clientVersion', null); + return is_string($v) ? $v : null; + } + + /** + * Get client engine (event metrics). + */ + public function getClientEngine(): ?string + { + $v = $this->getAttribute('clientEngine', null); + return is_string($v) ? $v : null; + } + + /** + * Get client engine version (event metrics). + */ + public function getClientEngineVersion(): ?string + { + $v = $this->getAttribute('clientEngineVersion', null); + return is_string($v) ? $v : null; + } + + /** + * Get device name (event metrics). + */ + public function getDeviceName(): ?string + { + $v = $this->getAttribute('deviceName', null); + return is_string($v) ? $v : null; + } + + /** + * Get device brand (event metrics). + */ + public function getDeviceBrand(): ?string + { + $v = $this->getAttribute('deviceBrand', null); + return is_string($v) ? $v : null; + } + + /** + * Get device model (event metrics). */ - // NOTE: loks0n flagged this as a leftover from the previous Metric - // implementation. Kept because tests (MetricTest, ClickHouseTest) and - // downstream consumers still call it; remove once those callers are - // migrated to direct `tags` attribute access. - public function getTags(): array + public function getDeviceModel(): ?string { - $tags = $this->getAttribute('tags', []); - return is_array($tags) ? $tags : []; + $v = $this->getAttribute('deviceModel', null); + return is_string($v) ? $v : null; } /** diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index 9ef9995..d2ff366 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -327,7 +327,6 @@ public function testConstructorInitializesWithData(): void 'status' => '201', 'resource' => 'bucket', 'resourceId' => 'abc123', - 'tags' => ['env' => 'prod'], ]; $metric = new Metric($data); @@ -341,7 +340,6 @@ public function testConstructorInitializesWithData(): void $this->assertEquals('201', $metric->getStatus()); $this->assertEquals('bucket', $metric->getResource()); $this->assertEquals('abc123', $metric->getResourceId()); - $this->assertEquals(['env' => 'prod'], $metric->getTags()); } /** @@ -458,25 +456,6 @@ public function testGetTimeReturnsNullWhenNotSet(): void $this->assertNull($metric->getTime()); } - /** - * Test Metric::getTags() returns tags - */ - public function testGetTagsReturnsTags(): void - { - $tags = ['region' => 'us-east', 'env' => 'prod']; - $metric = new Metric(['tags' => $tags]); - $this->assertEquals($tags, $metric->getTags()); - } - - /** - * Test Metric::getTags() returns empty array when not set - */ - public function testGetTagsReturnsEmptyArrayWhenNotSet(): void - { - $metric = new Metric([]); - $this->assertEquals([], $metric->getTags()); - } - /** * Test Metric::getTenant() returns tenant ID as string */ @@ -694,4 +673,44 @@ public function testGaugeSchemaHasTeamAndResourceColumns(): void } $this->assertNotContains('tags', $ids); } + + /** + * Test the 18 new typed getters return the value passed via the constructor. + */ + public function testNewGettersReturnString(): void + { + $m = new Metric([ + 'service' => 'storage', 'resourceInternalId' => '42', + 'teamId' => 'org_x', 'teamInternalId' => '7', + 'region' => 'fra', 'hostname' => 'app.example.com', + 'osCode' => 'IOS', 'osName' => 'iOS', 'osVersion' => '17.4', + 'clientType' => 'mobile-app', 'clientCode' => 'APW', 'clientName' => 'Appwrite SDK', + 'clientVersion' => '15.0.0', 'clientEngine' => 'WebKit', 'clientEngineVersion' => '605', + 'deviceName' => 'smartphone', 'deviceBrand' => 'Apple', 'deviceModel' => 'iPhone 13', + ]); + $this->assertSame('storage', $m->getService()); + $this->assertSame('42', $m->getResourceInternalId()); + $this->assertSame('org_x', $m->getTeamId()); + $this->assertSame('7', $m->getTeamInternalId()); + $this->assertSame('fra', $m->getRegion()); + $this->assertSame('app.example.com', $m->getHostname()); + $this->assertSame('IOS', $m->getOsCode()); + $this->assertSame('iOS', $m->getOsName()); + $this->assertSame('17.4', $m->getOsVersion()); + $this->assertSame('mobile-app', $m->getClientType()); + $this->assertSame('APW', $m->getClientCode()); + $this->assertSame('Appwrite SDK', $m->getClientName()); + $this->assertSame('15.0.0', $m->getClientVersion()); + $this->assertSame('WebKit', $m->getClientEngine()); + $this->assertSame('605', $m->getClientEngineVersion()); + $this->assertSame('smartphone', $m->getDeviceName()); + $this->assertSame('Apple', $m->getDeviceBrand()); + $this->assertSame('iPhone 13', $m->getDeviceModel()); + } + + public function testDroppedGettersDoNotExist(): void + { + $this->assertFalse(method_exists(Metric::class, 'getUserAgent')); + $this->assertFalse(method_exists(Metric::class, 'getTags')); + } } From cd556c9c3cad155aa66bd20901e4c85a98001ed9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:32:19 +0000 Subject: [PATCH 04/18] refactor(metric): drop tags special-case in validate() --- src/Usage/Metric.php | 8 -------- tests/Usage/MetricTest.php | 35 +++-------------------------------- 2 files changed, 3 insertions(+), 40 deletions(-) diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index 90772f8..c8ae88d 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -798,14 +798,6 @@ public static function validate(array $data, string $type = 'event'): void $value = $data[$attrId]; - // Special handling for tags: accept array (will be JSON-encoded) - if ($attrId === 'tags') { - if (!is_array($value)) { - throw new \Exception("Attribute '{$attrId}' must be an array, got " . gettype($value)); - } - continue; - } - // Validate based on attribute type match ($attrType) { 'string' => self::validateStringAttribute($attrId, $value, $size), diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index d2ff366..0c0b452 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -131,7 +131,7 @@ public function testValidateAcceptsValidEventData(): void 'status' => '201', 'resource' => 'bucket', 'resourceId' => 'abc123', - 'tags' => ['region' => 'us-east', 'env' => 'prod'], + 'region' => 'us', ]; // Should not throw exception @@ -148,7 +148,8 @@ public function testValidateAcceptsValidGaugeData(): void 'metric' => 'storage', 'value' => 10000, 'time' => '2024-01-01 12:00:00', - 'tags' => ['region' => 'us-east'], + 'teamId' => 'team1', + 'resourceId' => 'r1', ]; Metric::validate($validData, 'gauge'); @@ -282,36 +283,6 @@ public function testValidateRejectsInvalidDatetimeString(): void ], 'event'); } - /** - * Test Metric::validate() rejects non-array tags - */ - public function testValidateRejectsNonArrayTags(): void - { - $this->expectException(\Exception::class); - $this->expectExceptionMessage("Attribute 'tags' must be an array"); - - Metric::validate([ - 'metric' => 'requests', - 'value' => 100, - 'tags' => 'not-an-array', - ], 'event'); - } - - /** - * Test Metric::validate() accepts empty tags array - */ - public function testValidateAcceptsEmptyTags(): void - { - $data = [ - 'metric' => 'requests', - 'value' => 100, - 'tags' => [], - ]; - - Metric::validate($data, 'event'); - $this->assertTrue(true); - } - /** * Test Metric constructor initializes with data */ From 7ebff56c1594ec547cfd433e0ce63ca8a3d83409 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:33:08 +0000 Subject: [PATCH 05/18] feat(metric): bloom_filter indexes for new filterable columns --- src/Usage/Metric.php | 67 ++++++++++++++------------------------ tests/Usage/MetricTest.php | 49 ++++++++++++++++++---------- 2 files changed, 57 insertions(+), 59 deletions(-) diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index c8ae88d..d7778ec 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -692,46 +692,25 @@ public static function getSchema(): array */ public static function getEventIndexes(): array { - // Note: `metric` and `time` are part of the ClickHouse primary key - // (ORDER BY (tenant, metric, time, id)), so a separate bloom_filter - // index on either column would be redundant. They're omitted here. - return [ - [ - '$id' => 'index-path', - 'type' => 'key', - 'attributes' => ['path'], - ], - [ - '$id' => 'index-method', - 'type' => 'key', - 'attributes' => ['method'], - ], - [ - '$id' => 'index-status', - 'type' => 'key', - 'attributes' => ['status'], - ], - [ - '$id' => 'index-resource', - 'type' => 'key', - 'attributes' => ['resource'], - ], - [ - '$id' => 'index-resourceId', - 'type' => 'key', - 'attributes' => ['resourceId'], - ], - [ - '$id' => 'index-country', - 'type' => 'key', - 'attributes' => ['country'], - ], - [ - '$id' => 'index-userAgent', + // `metric` and `time` are part of the ClickHouse primary key + // (ORDER BY (tenant, metric, time, id)), so separate bloom_filter + // indexes on them would be redundant. + $indexed = [ + 'path', 'method', 'status', + 'service', 'resource', 'resourceId', 'resourceInternalId', + 'teamId', 'teamInternalId', + 'country', 'region', 'hostname', + 'osName', 'clientType', 'clientName', 'deviceName', + ]; + + return array_map( + static fn (string $col): array => [ + '$id' => 'index-' . $col, 'type' => 'key', - 'attributes' => ['userAgent'], + 'attributes' => [$col], ], - ]; + $indexed, + ); } /** @@ -741,10 +720,14 @@ public static function getEventIndexes(): array */ public static function getGaugeIndexes(): array { - // `metric` and `time` are part of the primary key; no secondary - // indexes needed. Returning an empty array keeps the table DDL - // free of redundant bloom_filter clauses. - return []; + return array_map( + static fn (string $col): array => [ + '$id' => 'index-' . $col, + 'type' => 'key', + 'attributes' => [$col], + ], + ['resourceId', 'resourceInternalId', 'teamId', 'teamInternalId'], + ); } /** diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index 0c0b452..caa5b23 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -72,39 +72,54 @@ public function testGetSchemaReturnsEventSchema(): void } /** - * Test Metric::getEventIndexes() returns correct index definitions + * Test Metric::getEventIndexes() returns one entry per indexed dimension. */ public function testGetEventIndexesReturnsIndexDefinitions(): void { $indexes = Metric::getEventIndexes(); $this->assertIsArray($indexes); - // metric/time are now covered by the primary key (ORDER BY (tenant, - // metric, time, id)), so only event-specific secondary indexes - // remain: path, method, status, resource, resourceId, country, - // userAgent. - $this->assertCount(7, $indexes); - $this->assertEquals('index-path', $indexes[0]['$id']); - $this->assertEquals('index-method', $indexes[1]['$id']); - $this->assertEquals('index-status', $indexes[2]['$id']); - $this->assertEquals('index-resource', $indexes[3]['$id']); - $this->assertEquals('index-resourceId', $indexes[4]['$id']); - $this->assertEquals('index-country', $indexes[5]['$id']); - $this->assertEquals('index-userAgent', $indexes[6]['$id']); + $ids = array_column($indexes, '$id'); + $this->assertNotContains('index-userAgent', $ids, 'userAgent index must be dropped'); } /** - * Test Metric::getGaugeIndexes() returns correct index definitions + * Test Metric::getGaugeIndexes() returns one entry per gauge dimension. */ public function testGetGaugeIndexesReturnsIndexDefinitions(): void { $indexes = Metric::getGaugeIndexes(); - // Gauges only filter by metric and time, both in the primary key, - // so no secondary indexes are needed. $this->assertIsArray($indexes); - $this->assertCount(0, $indexes); + $this->assertCount(count(Metric::GAUGE_COLUMNS), $indexes); + } + + public function testEventIndexesCoverNewFilterableColumns(): void + { + $indexed = []; + foreach (Metric::getEventIndexes() as $idx) { + $indexed = array_merge($indexed, $idx['attributes']); + } + foreach ([ + 'path', 'method', 'status', 'service', 'resource', + 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', + 'country', 'region', 'hostname', + 'osName', 'clientType', 'clientName', 'deviceName', + ] as $col) { + $this->assertContains($col, $indexed, "Event indexes missing {$col}"); + } + } + + public function testGaugeIndexesCoverIdColumns(): void + { + $indexed = []; + foreach (Metric::getGaugeIndexes() as $idx) { + $indexed = array_merge($indexed, $idx['attributes']); + } + foreach (['resourceId', 'resourceInternalId', 'teamId', 'teamInternalId'] as $col) { + $this->assertContains($col, $indexed); + } } /** From 321c8d7c553bc005b16a6dff1f4e6915cf0ad0e6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:33:21 +0000 Subject: [PATCH 06/18] feat(clickhouse): widen LowCardinality treatment for bounded vocab columns --- src/Usage/Adapter/ClickHouse.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index f0f67dc..ae3af5e 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1248,8 +1248,13 @@ private function getColumnType(string $id, string $type = 'event'): string throw new Exception("Attribute {$id} not found in {$type} schema"); } - // Country uses LowCardinality for efficient storage of low-cardinality values - if ($id === 'country') { + $lowCardinality = [ + 'country', 'region', 'service', 'resource', + 'osCode', 'osName', 'clientType', 'clientCode', 'clientName', + 'clientEngine', 'deviceName', 'deviceBrand', + ]; + + if (in_array($id, $lowCardinality, true)) { return 'LowCardinality(Nullable(String))'; } From c9676c6360866dbb921e69c4b77068d7c4c7b43e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:34:04 +0000 Subject: [PATCH 07/18] feat(clickhouse): widen daily MV with resource/team dimensions --- src/Usage/Adapter/ClickHouse.php | 38 +++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index ae3af5e..a07b03e 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1072,6 +1072,11 @@ private function createDailyTable(): void 'metric String', 'value Int64', 'time DateTime64(3)', + 'resource LowCardinality(Nullable(String))', + 'resourceId Nullable(String)', + 'resourceInternalId Nullable(String)', + 'teamId Nullable(String)', + 'teamInternalId Nullable(String)', ]; if ($this->sharedTables) { @@ -1080,9 +1085,9 @@ private function createDailyTable(): void $columnDefs = implode(",\n ", $columns); - // metric and time are part of the ORDER BY (primary key) — no - // secondary bloom_filter indexes needed. - $dailyOrderBy = $this->sharedTables ? '(tenant, metric, time)' : '(metric, time)'; + $dailyOrderBy = $this->sharedTables + ? '(tenant, metric, time, resource, resourceId, resourceInternalId, teamId, teamInternalId)' + : '(metric, time, resource, resourceId, resourceInternalId, teamId, teamInternalId)'; $createDailyTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDailyTable} ( @@ -1113,17 +1118,19 @@ private function createDailyMaterializedView(): void $dailyMvName = $this->getTableName() . '_events_daily_mv'; $escapedEventsTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($eventsTable); - $escapedDailyTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyTableName); - $escapedDailyMv = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyMvName); + $escapedDailyTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyTableName); + $escapedDailyMv = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyMvName); + + $dimensions = 'resource, resourceId, resourceInternalId, teamId, teamInternalId'; if ($this->sharedTables) { - $innerSelect = "metric, tenant, sum(value) as value, toStartOfDay(time) as d"; - $innerGroupBy = "metric, tenant, d"; - $outerSelect = "metric, value, d as time, tenant"; + $innerSelect = "metric, tenant, {$dimensions}, sum(value) as value, toStartOfDay(time) as d"; + $innerGroupBy = "metric, tenant, {$dimensions}, d"; + $outerSelect = "metric, value, d as time, tenant, {$dimensions}"; } else { - $innerSelect = "metric, sum(value) as value, toStartOfDay(time) as d"; - $innerGroupBy = "metric, d"; - $outerSelect = "metric, value, d as time"; + $innerSelect = "metric, {$dimensions}, sum(value) as value, toStartOfDay(time) as d"; + $innerGroupBy = "metric, {$dimensions}, d"; + $outerSelect = "metric, value, d as time, {$dimensions}"; } $createDailyMvSql = " @@ -1175,7 +1182,11 @@ private function validateAttributeName(string $attributeName, string $type = 'ev /** * Columns available in the events daily (pre-aggregated) table. */ - private const DAILY_COLUMNS = ['metric', 'value', 'time']; + private const DAILY_COLUMNS = [ + 'metric', 'value', 'time', + 'resource', 'resourceId', 'resourceInternalId', + 'teamId', 'teamInternalId', + ]; /** * Validate that a query attribute exists in the daily table schema. @@ -1197,9 +1208,10 @@ private function validateDailyAttributeName(string $attributeName): bool return true; } + $allowed = implode(', ', self::DAILY_COLUMNS) . ($this->sharedTables ? ', tenant' : ''); throw new Exception( "Invalid attribute '{$attributeName}' for daily table. " - . "Only metric, value, time" . ($this->sharedTables ? ", tenant" : "") . " are available." + . "Allowed: {$allowed}." ); } From 5c044a363087686d727b5cbb48c5a47be84dc1b5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:34:23 +0000 Subject: [PATCH 08/18] feat(clickhouse): strict tag extraction, drop tags column, normalize country/region --- src/Usage/Adapter/ClickHouse.php | 59 +++++++++++++++----------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index a07b03e..cac169e 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1405,42 +1405,37 @@ public function addBatch(array $metrics, string $type, int $batchSize = self::IN $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; - if ($type === Usage::TYPE_EVENT) { - // Extract event-specific columns from tags into dedicated columns - $eventColumns = []; - foreach (Metric::EVENT_COLUMNS as $col) { - if (isset($tags[$col])) { - $tagValue = $tags[$col]; - $eventColumns[$col] = is_string($tagValue) ? $tagValue : (is_scalar($tagValue) ? (string) $tagValue : null); - unset($tags[$col]); - } else { - $eventColumns[$col] = null; - } + $allowed = $type === Usage::TYPE_EVENT ? Metric::EVENT_COLUMNS : Metric::GAUGE_COLUMNS; + + $columns = []; + foreach ($allowed as $col) { + $val = $tags[$col] ?? null; + unset($tags[$col]); + if (is_string($val)) { + $val = $val === '' ? null : $val; + } elseif (is_scalar($val)) { + $val = (string) $val; + } else { + $val = null; } + if (($col === 'country' || $col === 'region') && is_string($val)) { + $val = strtolower($val); + } + $columns[$col] = $val; + } - ksort($tags); - - $row = array_merge([ - 'id' => $this->generateId(), - 'metric' => $metric, - 'value' => $value, - 'time' => $this->formatDateTime(null), - ], $eventColumns, [ - 'tags' => $tags, - ]); - } else { - // Gauge: simple schema - ksort($tags); - - $row = [ - 'id' => $this->generateId(), - 'metric' => $metric, - 'value' => $value, - 'time' => $this->formatDateTime(null), - 'tags' => $tags, - ]; + if (!empty($tags)) { + $unknown = array_key_first($tags); + throw new Exception("Unknown column '{$unknown}' for {$type}"); } + $row = array_merge([ + 'id' => $this->generateId(), + 'metric' => $metric, + 'value' => $value, + 'time' => $this->formatDateTime(null), + ], $columns); + if ($this->sharedTables) { $row['tenant'] = $tenant; } From 1e68d6e8540a994583cd84d6b20941acf155b0e8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:35:00 +0000 Subject: [PATCH 09/18] feat(database-adapter): apply schema refactor to dev adapter --- src/Usage/Adapter/Database.php | 43 ++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 0f5e204..8dc3b7c 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -73,7 +73,10 @@ public function setup(): void throw new \Exception('You need to create the database before running Usage setup'); } - // Use event attributes which is a superset (includes path/method/status/resource/resourceId) + // Event schema is a superset of the gauge schema for the dimensions + // that exist in both (resourceId, resourceInternalId, teamId, + // teamInternalId), so a single Database collection backed by the + // event schema works for both types. $attributes = $this->getAttributeDocuments('event'); $indexDocs = $this->getIndexDocuments('event'); @@ -138,25 +141,41 @@ public function addBatch(array $metrics, string $type, int $batchSize = 1000): b throw new \InvalidArgumentException('Value cannot be negative'); } + /** @var array $tags */ $tags = $metric['tags'] ?? []; - ksort($tags); - $docData = [ + $allowed = $type === Usage::TYPE_EVENT ? Metric::EVENT_COLUMNS : Metric::GAUGE_COLUMNS; + + $columns = []; + foreach ($allowed as $col) { + $val = $tags[$col] ?? null; + unset($tags[$col]); + if (is_string($val)) { + $val = $val === '' ? null : $val; + } elseif (is_scalar($val)) { + $val = (string) $val; + } else { + $val = null; + } + if (($col === 'country' || $col === 'region') && is_string($val)) { + $val = strtolower($val); + } + $columns[$col] = $val; + } + + if (!empty($tags)) { + $unknown = array_key_first($tags); + throw new \Exception("Unknown column '{$unknown}' for {$type}"); + } + + $docData = array_merge([ '$id' => $this->generateId(), '$permissions' => [], 'metric' => $metric['metric'], 'value' => $metric['value'], 'type' => $type, 'time' => (new \DateTime())->format('Y-m-d H:i:s.v'), - 'tags' => $tags, - ]; - - // For events, extract event-specific columns from tags - if ($type === Usage::TYPE_EVENT) { - foreach (Metric::EVENT_COLUMNS as $col) { - $docData[$col] = $tags[$col] ?? null; - } - } + ], $columns); $documents[] = new Document($docData); } From e29a1c9e33e7ee35752a50c2e42e175cf47a3c92 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:40:05 +0000 Subject: [PATCH 10/18] test(clickhouse): cover new schema columns, strict tags, normalization --- tests/Usage/Adapter/ClickHouseTest.php | 205 +++++++++++++++++++++---- tests/Usage/UsageBase.php | 2 +- 2 files changed, 177 insertions(+), 30 deletions(-) diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index 81d3d14..bbd35da 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -129,7 +129,7 @@ public function testLargeBatchWithSmallBatchSize(): void $metrics[] = [ 'metric' => 'large-batch-metric', 'value' => $i, - 'tags' => ['index' => (string) $i], + 'tags' => ['resourceId' => (string) $i], ]; } @@ -210,7 +210,9 @@ public function testBatchWithTagsClickHouse(): void } /** - * Test event-specific columns are extracted from tags + * Test event-specific columns are extracted from tags into dedicated columns + * and surfaced via the new typed getters. Verifies the full dimension set + * round-trips through ClickHouse. */ public function testEventColumnsExtractedFromTags(): void { @@ -224,11 +226,27 @@ public function testEventColumnsExtractedFromTags(): void 'path' => '/v1/storage/files', 'method' => 'POST', 'status' => '201', + 'service' => 'storage', 'resource' => 'bucket', 'resourceId' => 'bucket123', + 'resourceInternalId' => '42', + 'teamId' => 'team_x', + 'teamInternalId' => '7', 'country' => 'US', - 'userAgent' => 'test-agent', 'region' => 'us-east', + 'hostname' => 'app.example.com', + 'osCode' => 'IOS', + 'osName' => 'iOS', + 'osVersion' => '17.4', + 'clientType' => 'mobile-app', + 'clientCode' => 'APW', + 'clientName' => 'Appwrite SDK', + 'clientVersion' => '15.0.0', + 'clientEngine' => 'WebKit', + 'clientEngineVersion' => '605', + 'deviceName' => 'smartphone', + 'deviceBrand' => 'Apple', + 'deviceModel' => 'iPhone 13', ], ], ]; @@ -242,25 +260,155 @@ public function testEventColumnsExtractedFromTags(): void $this->assertCount(1, $results); $metric = $results[0]; - // Event-specific columns should be set $this->assertEquals('/v1/storage/files', $metric->getPath()); $this->assertEquals('POST', $metric->getMethod()); $this->assertEquals('201', $metric->getStatus()); + $this->assertEquals('storage', $metric->getService()); $this->assertEquals('bucket', $metric->getResource()); $this->assertEquals('bucket123', $metric->getResourceId()); - $this->assertEquals('US', $metric->getCountry()); - $this->assertEquals('test-agent', $metric->getUserAgent()); - - // Remaining tags should only contain non-event fields - $tags = $metric->getTags(); - $this->assertEquals('us-east', $tags['region'] ?? null); - $this->assertArrayNotHasKey('path', $tags); - $this->assertArrayNotHasKey('method', $tags); - $this->assertArrayNotHasKey('status', $tags); - $this->assertArrayNotHasKey('resource', $tags); - $this->assertArrayNotHasKey('resourceId', $tags); - $this->assertArrayNotHasKey('country', $tags); - $this->assertArrayNotHasKey('userAgent', $tags); + $this->assertEquals('42', $metric->getResourceInternalId()); + $this->assertEquals('team_x', $metric->getTeamId()); + $this->assertEquals('7', $metric->getTeamInternalId()); + // country and region are lowercased on write + $this->assertEquals('us', $metric->getCountry()); + $this->assertEquals('us-east', $metric->getRegion()); + $this->assertEquals('app.example.com', $metric->getHostname()); + $this->assertEquals('IOS', $metric->getOsCode()); + $this->assertEquals('iOS', $metric->getOsName()); + $this->assertEquals('17.4', $metric->getOsVersion()); + $this->assertEquals('mobile-app', $metric->getClientType()); + $this->assertEquals('APW', $metric->getClientCode()); + $this->assertEquals('Appwrite SDK', $metric->getClientName()); + $this->assertEquals('15.0.0', $metric->getClientVersion()); + $this->assertEquals('WebKit', $metric->getClientEngine()); + $this->assertEquals('605', $metric->getClientEngineVersion()); + $this->assertEquals('smartphone', $metric->getDeviceName()); + $this->assertEquals('Apple', $metric->getDeviceBrand()); + $this->assertEquals('iPhone 13', $metric->getDeviceModel()); + } + + /** + * Gauge rows round-trip the four gauge dimension columns. + */ + public function testGaugeColumnsRoundTrip(): void + { + $this->usage->purge([], Usage::TYPE_GAUGE); + + $this->assertTrue($this->usage->addBatch([ + [ + 'metric' => 'gauge-cols-test', + 'value' => 500, + 'tags' => [ + 'teamId' => 'team_x', + 'teamInternalId' => '7', + 'resourceId' => 'r1', + 'resourceInternalId' => '42', + ], + ], + ], Usage::TYPE_GAUGE)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['gauge-cols-test']), + ], Usage::TYPE_GAUGE); + + $this->assertCount(1, $results); + $metric = $results[0]; + $this->assertEquals('team_x', $metric->getTeamId()); + $this->assertEquals('7', $metric->getTeamInternalId()); + $this->assertEquals('r1', $metric->getResourceId()); + $this->assertEquals('42', $metric->getResourceInternalId()); + } + + public function testUnknownTagKeyThrows(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessageMatches("/Unknown column 'bogus'/"); + $this->usage->addBatch([ + ['metric' => 'x', 'value' => 1, 'tags' => ['bogus' => 'v']], + ], Usage::TYPE_EVENT); + } + + public function testCountryLowercased(): void + { + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'lc-country', 'value' => 1, 'tags' => ['country' => 'US']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['lc-country']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertSame('us', $results[0]->getCountry()); + } + + public function testRegionLowercased(): void + { + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'lc-region', 'value' => 1, 'tags' => ['region' => 'FR']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['lc-region']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertSame('fr', $results[0]->getRegion()); + } + + public function testEmptyStringCoercedToNull(): void + { + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'empty-string', 'value' => 1, 'tags' => ['osName' => '']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['empty-string']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertNull($results[0]->getOsName()); + } + + /** + * Round-trip a row that exercises every queryable dimension column to + * confirm the events table schema accepts and persists each one. + */ + public function testEventsSchemaPersistsAllNewColumns(): void + { + $this->usage->purge([], Usage::TYPE_EVENT); + + $tags = [ + 'path' => '/v1/x', 'method' => 'GET', 'status' => '200', + 'service' => 'storage', 'resource' => 'bucket', + 'resourceId' => 'r1', 'resourceInternalId' => '42', + 'teamId' => 't1', 'teamInternalId' => '7', + 'country' => 'us', 'region' => 'fra', 'hostname' => 'h.example.com', + 'osCode' => 'IOS', 'osName' => 'iOS', 'osVersion' => '17.4', + 'clientType' => 'browser', 'clientCode' => 'CH', + 'clientName' => 'Chrome', 'clientVersion' => '125', + 'clientEngine' => 'Blink', 'clientEngineVersion' => '125', + 'deviceName' => 'desktop', 'deviceBrand' => 'Apple', + 'deviceModel' => 'MacBook', + ]; + + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'schema-roundtrip', 'value' => 1, 'tags' => $tags], + ], Usage::TYPE_EVENT)); + + // Filtering on each indexed dimension should be schema-valid. + foreach (['service', 'resourceInternalId', 'teamId', 'teamInternalId', 'region', 'hostname', 'osName', 'clientName', 'deviceName'] as $col) { + $value = $tags[$col]; + $expected = ($col === 'country' || $col === 'region') ? strtolower($value) : $value; + $rows = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['schema-roundtrip']), + \Utopia\Query\Query::equal($col, [$expected]), + ], Usage::TYPE_EVENT); + $this->assertGreaterThanOrEqual(1, count($rows), "Filter on {$col} returned no rows"); + } } /** @@ -317,7 +465,7 @@ public function testGaugeTableSimpleSchema(): void $this->usage->purge([], Usage::TYPE_GAUGE); $this->assertTrue($this->usage->addBatch([ - ['metric' => 'gauge-simple', 'value' => 500, 'tags' => ['region' => 'us-east']], + ['metric' => 'gauge-simple', 'value' => 500, 'tags' => ['resourceId' => 'r1']], ], Usage::TYPE_GAUGE)); $results = $this->usage->find([ @@ -333,7 +481,7 @@ public function testGaugeTableSimpleSchema(): void $this->assertNull($results[0]->getMethod()); $this->assertNull($results[0]->getStatus()); $this->assertNull($results[0]->getResource()); - $this->assertNull($results[0]->getResourceId()); + $this->assertEquals('r1', $results[0]->getResourceId()); } /** @@ -430,7 +578,7 @@ public function testMetricsWithSpecialCharacters(): void { $specialVal = "Text with \n newline, \t tab, \"quote\", and unicode \u{1F600}"; $this->assertTrue($this->usage->addBatch([ - ['metric' => 'special-metric', 'value' => 1, 'tags' => ['s' => $specialVal]], + ['metric' => 'special-metric', 'value' => 1, 'tags' => ['hostname' => $specialVal]], ], Usage::TYPE_EVENT)); $results = $this->usage->find([ @@ -439,8 +587,7 @@ public function testMetricsWithSpecialCharacters(): void $this->assertEquals(1, count($results)); $this->assertEquals('special-metric', $results[0]->getMetric()); - $tags = $results[0]->getTags(); - $this->assertEquals($specialVal, $tags['s']); + $this->assertEquals($specialVal, $results[0]->getHostname()); } /** @@ -453,10 +600,10 @@ public function testFindComprehensive(): void // Setup test data $this->usage->addBatch([ - ['metric' => 'metric-A', 'value' => 10, 'tags' => ['category' => 'cat1']], + ['metric' => 'metric-A', 'value' => 10, 'tags' => ['service' => 'cat1']], ], Usage::TYPE_EVENT); $this->usage->addBatch([ - ['metric' => 'metric-B', 'value' => 20, 'tags' => ['category' => 'cat2']], + ['metric' => 'metric-B', 'value' => 20, 'tags' => ['service' => 'cat2']], ], Usage::TYPE_EVENT); // 1. Array Equal (IN) @@ -707,9 +854,9 @@ public function testCompression(): void // Insert data using addBatch with compression enabled $batchResult = $usage->addBatch([ - ['metric' => 'compression.test.batch', 'value' => 50, 'tags' => ['type' => 'batch']], - ['metric' => 'compression.test.batch', 'value' => 75, 'tags' => ['type' => 'batch']], - ['metric' => 'compression.test.single', 'value' => 100, 'tags' => ['type' => 'single']], + ['metric' => 'compression.test.batch', 'value' => 50, 'tags' => ['service' => 'batch']], + ['metric' => 'compression.test.batch', 'value' => 75, 'tags' => ['service' => 'batch']], + ['metric' => 'compression.test.single', 'value' => 100, 'tags' => ['service' => 'single']], ], Usage::TYPE_EVENT); $this->assertTrue($batchResult); @@ -774,7 +921,7 @@ public function testConnectionPooling(): void // Make some requests $usage->addBatch([ - ['metric' => 'pooling.test', 'value' => 100, 'tags' => ['test' => 'value']], + ['metric' => 'pooling.test', 'value' => 100, 'tags' => ['service' => 'value']], ], Usage::TYPE_EVENT); $usage->find([], Usage::TYPE_EVENT); $usage->count([], Usage::TYPE_EVENT); @@ -885,7 +1032,7 @@ public function testRetryWithSuccessfulOperations(): void // These operations should succeed on first attempt (no retries needed) $result = $usage->addBatch([ - ['metric' => 'retry.test', 'value' => 100, 'tags' => ['test' => 'success']], + ['metric' => 'retry.test', 'value' => 100, 'tags' => ['service' => 'success']], ], Usage::TYPE_EVENT); $this->assertTrue($result); diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index 6bb71d0..e53f0fd 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -34,7 +34,7 @@ public function createUsageMetrics(): void // Gauges: point-in-time snapshots $this->assertTrue($this->usage->addBatch([ - ['metric' => 'storage', 'value' => 10000, 'tags' => ['region' => 'us-east']], + ['metric' => 'storage', 'value' => 10000, 'tags' => ['resourceId' => 'p1']], ], Usage::TYPE_GAUGE)); } From 1d6736283165d4b4b2776a563d26edfe0f8d4af6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:41:21 +0000 Subject: [PATCH 11/18] test(database-adapter): cover new schema columns and strict tags --- tests/Usage/Adapter/DatabaseTest.php | 159 ++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 4 deletions(-) diff --git a/tests/Usage/Adapter/DatabaseTest.php b/tests/Usage/Adapter/DatabaseTest.php index 5e0c05a..779450c 100644 --- a/tests/Usage/Adapter/DatabaseTest.php +++ b/tests/Usage/Adapter/DatabaseTest.php @@ -21,8 +21,12 @@ class DatabaseTest extends TestCase protected function initializeUsage(): void { - $dbHost = 'mariadb'; - $dbPort = '3306'; + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed in this environment'); + } + + $dbHost = getenv('MARIADB_HOST') ?: 'mariadb'; + $dbPort = getenv('MARIADB_PORT') ?: '3306'; $dbUser = 'root'; $dbPass = 'password'; @@ -50,6 +54,149 @@ protected function initializeUsage(): void } } + /** + * Round-trip a row with the full event dimension set through the + * Database adapter. + */ + public function testEventColumnsExtractedFromTags(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->usage->purge([], Usage::TYPE_EVENT); + + $this->assertTrue($this->usage->addBatch([ + [ + 'metric' => 'event-cols-db', + 'value' => 42, + 'tags' => [ + 'path' => '/v1/storage/files', + 'method' => 'POST', + 'status' => '201', + 'service' => 'storage', + 'resource' => 'bucket', + 'resourceId' => 'bucket123', + 'resourceInternalId' => '42', + 'teamId' => 'team_x', + 'teamInternalId' => '7', + 'country' => 'US', + 'region' => 'us-east', + 'hostname' => 'app.example.com', + 'osName' => 'iOS', + 'clientName' => 'Appwrite SDK', + 'deviceName' => 'smartphone', + ], + ], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['event-cols-db']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $metric = $results[0]; + $this->assertEquals('/v1/storage/files', $metric->getPath()); + $this->assertEquals('storage', $metric->getService()); + $this->assertEquals('42', $metric->getResourceInternalId()); + $this->assertEquals('team_x', $metric->getTeamId()); + $this->assertEquals('7', $metric->getTeamInternalId()); + $this->assertEquals('us', $metric->getCountry()); + $this->assertEquals('us-east', $metric->getRegion()); + $this->assertEquals('app.example.com', $metric->getHostname()); + $this->assertEquals('iOS', $metric->getOsName()); + $this->assertEquals('Appwrite SDK', $metric->getClientName()); + $this->assertEquals('smartphone', $metric->getDeviceName()); + } + + /** + * Gauge rows round-trip the four gauge dimension columns. + */ + public function testGaugeColumnsRoundTrip(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->usage->purge([], Usage::TYPE_GAUGE); + + $this->assertTrue($this->usage->addBatch([ + [ + 'metric' => 'gauge-cols-db', + 'value' => 500, + 'tags' => [ + 'teamId' => 'team_x', + 'teamInternalId' => '7', + 'resourceId' => 'r1', + 'resourceInternalId' => '42', + ], + ], + ], Usage::TYPE_GAUGE)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['gauge-cols-db']), + ], Usage::TYPE_GAUGE); + + $this->assertCount(1, $results); + $metric = $results[0]; + $this->assertEquals('team_x', $metric->getTeamId()); + $this->assertEquals('7', $metric->getTeamInternalId()); + $this->assertEquals('r1', $metric->getResourceId()); + $this->assertEquals('42', $metric->getResourceInternalId()); + } + + public function testUnknownTagKeyThrows(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->expectException(\Exception::class); + $this->expectExceptionMessageMatches("/Unknown column 'bogus'/"); + $this->usage->addBatch([ + ['metric' => 'x', 'value' => 1, 'tags' => ['bogus' => 'v']], + ], Usage::TYPE_EVENT); + } + + public function testCountryAndRegionLowercased(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'lc-db', 'value' => 1, 'tags' => ['country' => 'US', 'region' => 'FR']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['lc-db']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertSame('us', $results[0]->getCountry()); + $this->assertSame('fr', $results[0]->getRegion()); + } + + public function testEmptyStringCoercedToNull(): void + { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + + $this->usage->purge([], Usage::TYPE_EVENT); + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'empty-db', 'value' => 1, 'tags' => ['osName' => '']], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->find([ + \Utopia\Query\Query::equal('metric', ['empty-db']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $results); + $this->assertNull($results[0]->getOsName()); + } + /** * Test healthCheck() method */ @@ -78,9 +225,13 @@ public function testHealthCheck(): void */ public function testHealthCheckWithNonExistentDatabase(): void { + if (!extension_loaded('pdo_mysql')) { + $this->markTestSkipped('pdo_mysql extension is not installed'); + } + // Create a new database instance pointing to a non-existent database - $dbHost = 'mariadb'; - $dbPort = '3306'; + $dbHost = getenv('MARIADB_HOST') ?: 'mariadb'; + $dbPort = getenv('MARIADB_PORT') ?: '3306'; $dbUser = 'root'; $dbPass = 'password'; From 90298f605c3b857e786a5ab0882dcd8d8c2a74a3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:42:54 +0000 Subject: [PATCH 12/18] docs(usage): refresh README and Metric docs for new schema --- README.md | 72 ++++++++++++++++++++++++++++++++++---------- src/Usage/Metric.php | 31 ++++++++++++++----- 2 files changed, 80 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index eb92d96..c3e5f9b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Although this library is part of the [Utopia Framework](https://github.com/utopi ## Features - **Two Table Architecture**: Separate events and gauges tables optimized for their access patterns -- **Events Table**: Request-level metrics with dedicated columns (path, method, status, resource, country, userAgent) +- **Events Table**: Request-level metrics with dedicated columns for every dimension we filter on (path/method/status, service/resource ids, team ids, country/region, hostname, parsed UA fields) - **Gauges Table**: Simple resource snapshots (storage size, user count, etc.) - **Query-Time Aggregation**: No write-time period fan-out — aggregate by any interval at query time - **Daily Materialized View**: Pre-aggregated daily SummingMergeTree for fast billing queries @@ -72,7 +72,12 @@ $usage->setup(); Events are request-level metrics like bandwidth, executions, API calls. They are summed when aggregated. -Event-specific columns: `path`, `method`, `status`, `resource`, `resourceId`, `country`, `userAgent` +Event-specific columns (see `Metric::EVENT_COLUMNS`): `path`, `method`, `status`, +`service`, `resource`, `resourceId`, `resourceInternalId`, `teamId`, +`teamInternalId`, `country`, `region`, `hostname`, `osCode`, `osName`, +`osVersion`, `clientType`, `clientCode`, `clientName`, `clientVersion`, +`clientEngine`, `clientEngineVersion`, `deviceName`, `deviceBrand`, +`deviceModel`. ```php // Collect events — values accumulate in-memory buffer (summed per metric) @@ -80,26 +85,43 @@ $usage->collect('bandwidth', 5000, Usage::TYPE_EVENT, [ 'path' => '/v1/storage/files', 'method' => 'POST', 'status' => '201', + 'service' => 'storage', 'resource' => 'bucket', 'resourceId' => 'abc123', + 'resourceInternalId' => '42', + 'teamId' => 'team_x', + 'teamInternalId' => '7', 'country' => 'US', - 'userAgent' => 'AppwriteSDK/1.0', + 'region' => 'us-east', + 'hostname' => 'app.example.com', + 'osName' => 'iOS', + 'clientName' => 'Appwrite SDK', + 'deviceName' => 'smartphone', ]); - -// Event columns are auto-extracted from tags into dedicated columns -// Remaining tags stay in the JSON tags column ``` +### Strict tag keys + +All keys passed in the `tags` array must map to a known event or gauge +column (see `Metric::EVENT_COLUMNS` and `Metric::GAUGE_COLUMNS`). Unknown +keys throw at write time — there is no JSON catch-all. To add a new +dimension, widen the schema and bump the library version. + ### Gauges (Point-in-Time) Gauges are resource snapshots like storage size, user count, file count. Last-write-wins semantics. +Gauge-specific columns (see `Metric::GAUGE_COLUMNS`): `teamId`, +`teamInternalId`, `resourceId`, `resourceInternalId`. + ```php // Collect gauges — last value wins per metric in buffer $usage->collect('users', 1500, Usage::TYPE_GAUGE); $usage->collect('storage.size', 1048576, Usage::TYPE_GAUGE, [ - 'resource' => 'bucket', + 'teamId' => 'team_x', + 'teamInternalId' => '7', 'resourceId' => 'abc123', + 'resourceInternalId' => '42', ]); ``` @@ -121,12 +143,12 @@ $usage->setFlushInterval(10); // Flush after 10 seconds (default: 20) ```php // Write directly without buffering $usage->addBatch([ - ['metric' => 'requests', 'value' => 100, 'tags' => ['path' => '/v1/users']], - ['metric' => 'bandwidth', 'value' => 50000, 'tags' => ['country' => 'DE']], + ['metric' => 'requests', 'value' => 100, 'tags' => ['path' => '/v1/users', 'method' => 'GET']], + ['metric' => 'bandwidth', 'value' => 50000, 'tags' => ['country' => 'DE', 'region' => 'fra']], ], Usage::TYPE_EVENT); $usage->addBatch([ - ['metric' => 'users', 'value' => 42, 'tags' => []], + ['metric' => 'users', 'value' => 42, 'tags' => ['teamId' => 'team_x']], ], Usage::TYPE_GAUGE); ``` @@ -250,11 +272,21 @@ $usage->purge([], Usage::TYPE_GAUGE); | path | Nullable(String) | API endpoint path | | method | Nullable(String) | HTTP method | | status | Nullable(String) | HTTP status code | -| resource | Nullable(String) | Resource type | -| resourceId | Nullable(String) | Resource ID | -| country | LowCardinality(Nullable(String)) | ISO country code | -| userAgent | Nullable(String) | User agent string | -| tags | Nullable(String) | JSON for extra metadata | +| service | LowCardinality(Nullable(String)) | API service (storage, databases, …) | +| resource | LowCardinality(Nullable(String)) | Resource type (bucket, file, …) | +| resourceId | Nullable(String) | External resource id | +| resourceInternalId | Nullable(String) | Internal resource sequence | +| teamId | Nullable(String) | External team id | +| teamInternalId | Nullable(String) | Internal team sequence | +| country | LowCardinality(Nullable(String)) | ISO country code (lowercased) | +| region | LowCardinality(Nullable(String)) | Region code (lowercased) | +| hostname | Nullable(String) | Caller origin host | +| osCode, osName | LowCardinality(Nullable(String)) | Parsed OS short code / name | +| osVersion | Nullable(String) | Parsed OS version | +| clientType, clientCode, clientName, clientEngine | LowCardinality(Nullable(String)) | Parsed client identity | +| clientVersion, clientEngineVersion | Nullable(String) | Parsed client versions | +| deviceName, deviceBrand | LowCardinality(Nullable(String)) | Parsed device identity | +| deviceModel | Nullable(String) | Parsed device model | | tenant | Nullable(String) | Tenant ID (shared tables) | ### Gauges Table Schema @@ -265,7 +297,10 @@ $usage->purge([], Usage::TYPE_GAUGE); | metric | String | Metric name | | value | Int64 | Current value | | time | DateTime64(3) | Snapshot timestamp | -| tags | Nullable(String) | JSON metadata | +| teamId | Nullable(String) | External team id | +| teamInternalId | Nullable(String) | Internal team sequence | +| resourceId | Nullable(String) | External resource id | +| resourceInternalId | Nullable(String) | Internal resource sequence | | tenant | Nullable(String) | Tenant ID (shared tables) | ### Daily Table Schema @@ -275,6 +310,11 @@ $usage->purge([], Usage::TYPE_GAUGE); | metric | String | Metric name | | value | Int64 | Aggregated daily sum | | time | DateTime64(3) | Day start timestamp | +| resource | LowCardinality(Nullable(String)) | Resource type | +| resourceId | Nullable(String) | External resource id | +| resourceInternalId | Nullable(String) | Internal resource sequence | +| teamId | Nullable(String) | External team id | +| teamInternalId | Nullable(String) | Internal team sequence | | tenant | Nullable(String) | Tenant ID (shared tables) | ### Creating Custom Adapters diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index d7778ec..48e0250 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -23,9 +23,18 @@ * 'path' => '/v1/storage/files', * 'method' => 'POST', * 'status' => '201', + * 'service' => 'storage', * 'resource' => 'bucket', * 'resourceId' => 'abc123', - * 'tags' => ['region' => 'us-east', 'country' => 'US'] + * 'resourceInternalId' => '42', + * 'teamId' => 'team_x', + * 'teamInternalId' => '7', + * 'country' => 'us', + * 'region' => 'us-east', + * 'hostname' => 'app.example.com', + * 'osName' => 'iOS', + * 'clientName' => 'Appwrite SDK', + * 'deviceName' => 'smartphone', * ]); * * echo $metric->getMetric(); // 'bandwidth' @@ -65,14 +74,22 @@ class Metric extends ArrayObject * - metric: Name/type of the metric being tracked * - value: Numeric value of the metric * - time: Timestamp when the metric was recorded - * - path: API endpoint path (events only) - * - method: HTTP method (events only) - * - status: HTTP status code (events only) - * - resource: Resource type (events only) - * - resourceId: Resource ID (events only) - * - tags: Additional metadata as key-value pairs * - tenant: Tenant ID for multi-tenant environments * + * Event-only dimension columns (see EVENT_COLUMNS): + * - path / method / status: HTTP shape + * - service: API service segment (storage, databases, …) + * - resource / resourceId / resourceInternalId: resource identity + * - teamId / teamInternalId: owning team identity + * - country / region / hostname: geographic + caller origin + * - osCode / osName / osVersion: parsed user-agent OS fields + * - clientType / clientCode / clientName / clientVersion: parsed client + * - clientEngine / clientEngineVersion: parsed client engine + * - deviceName / deviceBrand / deviceModel: parsed device fields + * + * Gauge-only dimension columns (see GAUGE_COLUMNS): + * - teamId / teamInternalId / resourceId / resourceInternalId + * * @param array $input Metric data */ public function __construct(array $input = []) From cae2dcff4f5d1ea6d62980ed5c0a0cbe437fb677 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 May 2026 03:45:58 +0000 Subject: [PATCH 13/18] chore(usage): apply formatter and refresh PHPStan baseline --- tests/Usage/Adapter/ClickHouseTest.php | 2 +- tests/Usage/MetricTest.php | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index bbd35da..b9bd08e 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -402,7 +402,7 @@ public function testEventsSchemaPersistsAllNewColumns(): void // Filtering on each indexed dimension should be schema-valid. foreach (['service', 'resourceInternalId', 'teamId', 'teamInternalId', 'region', 'hostname', 'osName', 'clientName', 'deviceName'] as $col) { $value = $tags[$col]; - $expected = ($col === 'country' || $col === 'region') ? strtolower($value) : $value; + $expected = $col === 'region' ? strtolower($value) : $value; $rows = $this->usage->find([ \Utopia\Query\Query::equal('metric', ['schema-roundtrip']), \Utopia\Query\Query::equal($col, [$expected]), diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index caa5b23..955379e 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -99,7 +99,9 @@ public function testEventIndexesCoverNewFilterableColumns(): void { $indexed = []; foreach (Metric::getEventIndexes() as $idx) { - $indexed = array_merge($indexed, $idx['attributes']); + /** @var array $attrs */ + $attrs = $idx['attributes']; + $indexed = array_merge($indexed, $attrs); } foreach ([ 'path', 'method', 'status', 'service', 'resource', @@ -115,7 +117,9 @@ public function testGaugeIndexesCoverIdColumns(): void { $indexed = []; foreach (Metric::getGaugeIndexes() as $idx) { - $indexed = array_merge($indexed, $idx['attributes']); + /** @var array $attrs */ + $attrs = $idx['attributes']; + $indexed = array_merge($indexed, $attrs); } foreach (['resourceId', 'resourceInternalId', 'teamId', 'teamInternalId'] as $col) { $this->assertContains($col, $indexed); From 3639dc9a4c95e6f37d2dbfae592bbc527dd8cba2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 26 May 2026 03:59:35 +0000 Subject: [PATCH 14/18] fix: bump region column size to 64; share extraction helper across adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - region column was sized at 2 (copied from country); region codes like 'us-east' and 'eu-central-1' are routinely longer. Bumped to 64. - Extracted the tag→column extraction, normalisation, and strict unknown-key check into Metric::extractColumns() so ClickHouse and Database adapters share one implementation. --- src/Usage/Adapter/ClickHouse.php | 24 +---------------- src/Usage/Adapter/Database.php | 24 +---------------- src/Usage/Metric.php | 45 +++++++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 47 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index cac169e..b198ca1 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1405,29 +1405,7 @@ public function addBatch(array $metrics, string $type, int $batchSize = self::IN $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; - $allowed = $type === Usage::TYPE_EVENT ? Metric::EVENT_COLUMNS : Metric::GAUGE_COLUMNS; - - $columns = []; - foreach ($allowed as $col) { - $val = $tags[$col] ?? null; - unset($tags[$col]); - if (is_string($val)) { - $val = $val === '' ? null : $val; - } elseif (is_scalar($val)) { - $val = (string) $val; - } else { - $val = null; - } - if (($col === 'country' || $col === 'region') && is_string($val)) { - $val = strtolower($val); - } - $columns[$col] = $val; - } - - if (!empty($tags)) { - $unknown = array_key_first($tags); - throw new Exception("Unknown column '{$unknown}' for {$type}"); - } + $columns = Metric::extractColumns($tags, $type); $row = array_merge([ 'id' => $this->generateId(), diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 8dc3b7c..700e917 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -144,29 +144,7 @@ public function addBatch(array $metrics, string $type, int $batchSize = 1000): b /** @var array $tags */ $tags = $metric['tags'] ?? []; - $allowed = $type === Usage::TYPE_EVENT ? Metric::EVENT_COLUMNS : Metric::GAUGE_COLUMNS; - - $columns = []; - foreach ($allowed as $col) { - $val = $tags[$col] ?? null; - unset($tags[$col]); - if (is_string($val)) { - $val = $val === '' ? null : $val; - } elseif (is_scalar($val)) { - $val = (string) $val; - } else { - $val = null; - } - if (($col === 'country' || $col === 'region') && is_string($val)) { - $val = strtolower($val); - } - $columns[$col] = $val; - } - - if (!empty($tags)) { - $unknown = array_key_first($tags); - throw new \Exception("Unknown column '{$unknown}' for {$type}"); - } + $columns = Metric::extractColumns($tags, $type); $docData = array_merge([ '$id' => $this->generateId(), diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index 48e0250..4b286b8 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -616,7 +616,7 @@ public static function getEventSchema(): array $stringColumn('teamId', 255), $stringColumn('teamInternalId', 255), $stringColumn('country', 2), - $stringColumn('region', 2), + $stringColumn('region', 64), $stringColumn('hostname', 1024), $stringColumn('osCode', 256), $stringColumn('osName', 256), @@ -760,6 +760,49 @@ public static function getIndexes(): array return self::getEventIndexes(); } + /** + * Extract and normalize dimension columns from a tags array. + * + * For the given metric type ('event' or 'gauge'): + * - Pulls every known column out of $tags. + * - Coerces scalars to string, empty string to null. + * - Lowercases country and region. + * - Throws if $tags contains any unknown key (strict — no JSON catch-all). + * + * @param array $tags + * @param string $type 'event' or 'gauge' + * @return array + * @throws \Exception When an unknown column key is present in $tags. + */ + public static function extractColumns(array $tags, string $type): array + { + $allowed = $type === 'gauge' ? self::GAUGE_COLUMNS : self::EVENT_COLUMNS; + + $columns = []; + foreach ($allowed as $col) { + $val = $tags[$col] ?? null; + unset($tags[$col]); + if (is_string($val)) { + $val = $val === '' ? null : $val; + } elseif (is_scalar($val)) { + $val = (string) $val; + } else { + $val = null; + } + if (($col === 'country' || $col === 'region') && is_string($val)) { + $val = strtolower($val); + } + $columns[$col] = $val; + } + + if (!empty($tags)) { + $unknown = array_key_first($tags); + throw new \Exception("Unknown column '{$unknown}' for {$type}"); + } + + return $columns; + } + /** * Validate metric data against schema. * From ca09149b22a989a62428e1f0abeda123b26e6bdc Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 26 May 2026 04:47:12 +0000 Subject: [PATCH 15/18] fix(database): keep single-attribute indexes under MariaDB 768-byte limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hostname column 1024 -> 255. DNS hostnames are capped at 253 chars, so the wider size never gained anything but pushed the bloom_filter index past the InnoDB/utopia-php-database 768-byte single-attribute index prefix limit. - path column stays at 1024 for data fidelity (URLs with signed query strings exceed 255), but its index entry now uses an explicit lengths=[255] so MariaDB only indexes the first 255 chars. The index validator in utopia/database treats lengths[i] as the indexLength when set, taking the column under the 768 cap while preserving the full column for stored data. Caught by CI when the full Database adapter suite ran against MariaDB — local runs skipped 45 DatabaseTest cases because pdo_mysql wasn't loaded. Verified locally now via docker-compose: 195/195 passing. --- src/Usage/Metric.php | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index 4b286b8..e0ab988 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -617,7 +617,7 @@ public static function getEventSchema(): array $stringColumn('teamInternalId', 255), $stringColumn('country', 2), $stringColumn('region', 64), - $stringColumn('hostname', 1024), + $stringColumn('hostname', 255), $stringColumn('osCode', 256), $stringColumn('osName', 256), $stringColumn('osVersion', 255), @@ -721,11 +721,20 @@ public static function getEventIndexes(): array ]; return array_map( - static fn (string $col): array => [ - '$id' => 'index-' . $col, - 'type' => 'key', - 'attributes' => [$col], - ], + static function (string $col): array { + $entry = [ + '$id' => 'index-' . $col, + 'type' => 'key', + 'attributes' => [$col], + ]; + // path is sized 1024 for data fidelity; cap the index key at + // 255 so the MariaDB single-attribute index stays within the + // 768-byte InnoDB key prefix limit. + if ($col === 'path') { + $entry['lengths'] = [255]; + } + return $entry; + }, $indexed, ); } From aac7b95ff55b7ba5b34af8b3b8a9417700e76f09 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 27 May 2026 13:22:06 +0000 Subject: [PATCH 16/18] feat(usage-query): add groupBy method for dimensional aggregation Library-private extension that lives on UsageQuery (not the base utopia-php/query Query class) so cloud can express "group by service over 1d", "group by path x status over 1h" requests without forcing an SDK regeneration across every Appwrite client. - TYPE_GROUP_BY constant + static factory groupBy(attribute). - isMethod() accepts the new method so Query::parse() round-trips it from the JSON wire form. - isGroupBy / extractGroupBy / removeGroupBy helpers. extractGroupBy returns an array (multiple groupBy queries may coexist when bucketing by several dimensions at once), unlike extractGroupByInterval which is single-instance. - Unit tests cover the factory, isMethod, helpers, parse round-trip and the parsed-Query (base class) extraction path. --- src/Usage/UsageQuery.php | 61 +++++++++++++++++++- tests/Usage/UsageQueryTest.php | 100 +++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 2 deletions(-) diff --git a/src/Usage/UsageQuery.php b/src/Usage/UsageQuery.php index 586f922..60ce843 100644 --- a/src/Usage/UsageQuery.php +++ b/src/Usage/UsageQuery.php @@ -30,6 +30,7 @@ class UsageQuery extends Query { public const TYPE_GROUP_BY_INTERVAL = 'groupByInterval'; + public const TYPE_GROUP_BY = 'groupBy'; /** * Valid interval values and their ClickHouse INTERVAL equivalents. @@ -45,11 +46,11 @@ class UsageQuery extends Query ]; /** - * Override isMethod to accept groupByInterval in addition to all base Query methods. + * Override isMethod to accept groupByInterval and groupBy in addition to all base Query methods. */ public static function isMethod(string $value): bool { - if ($value === self::TYPE_GROUP_BY_INTERVAL) { + if ($value === self::TYPE_GROUP_BY_INTERVAL || $value === self::TYPE_GROUP_BY) { return true; } @@ -122,4 +123,60 @@ public static function removeGroupByInterval(array $queries): array return !self::isGroupByInterval($query); })); } + + /** + * Create a groupBy query for dimensional aggregation. + * + * Buckets results by the given attribute in addition to the time bucket + * supplied via `groupByInterval`. Multiple `groupBy` queries may be + * combined to bucket by several dimensions at once (e.g. service x status). + * + * @param string $attribute The dimension column to bucket on (service, path, status, ...). + * @return self + */ + public static function groupBy(string $attribute): self + { + return new self(self::TYPE_GROUP_BY, $attribute, []); + } + + /** + * Check if a query is a groupBy query. + * + * @param Query $query + * @return bool + */ + public static function isGroupBy(Query $query): bool + { + return $query->getMethod() === self::TYPE_GROUP_BY; + } + + /** + * Extract all groupBy queries from an array of queries. + * + * Multiple groupBy queries can coexist (group by service AND status), so + * this returns every match rather than the single-instance form used by + * groupByInterval. + * + * @param array $queries + * @return array + */ + public static function extractGroupBy(array $queries): array + { + return array_values(array_filter($queries, function (Query $query) { + return self::isGroupBy($query); + })); + } + + /** + * Remove all groupBy queries from an array of queries. + * + * @param array $queries + * @return array + */ + public static function removeGroupBy(array $queries): array + { + return array_values(array_filter($queries, function (Query $query) { + return !self::isGroupBy($query); + })); + } } diff --git a/tests/Usage/UsageQueryTest.php b/tests/Usage/UsageQueryTest.php index b4b07b1..31da415 100644 --- a/tests/Usage/UsageQueryTest.php +++ b/tests/Usage/UsageQueryTest.php @@ -131,4 +131,104 @@ public function testUsageQueryExtendsQuery(): void $query = UsageQuery::groupByInterval('time', '1h'); $this->assertInstanceOf(Query::class, $query); } + + public function testGroupByCreation(): void + { + $query = UsageQuery::groupBy('service'); + + $this->assertInstanceOf(UsageQuery::class, $query); + $this->assertEquals(UsageQuery::TYPE_GROUP_BY, $query->getMethod()); + $this->assertEquals('service', $query->getAttribute()); + $this->assertEquals([], $query->getValues()); + } + + public function testGroupByIsMethod(): void + { + $this->assertTrue(UsageQuery::isMethod(UsageQuery::TYPE_GROUP_BY)); + $this->assertTrue(UsageQuery::isMethod(UsageQuery::TYPE_GROUP_BY_INTERVAL)); + $this->assertTrue(UsageQuery::isMethod(Query::TYPE_EQUAL)); + $this->assertFalse(UsageQuery::isMethod('notARealMethod')); + } + + public function testIsGroupBy(): void + { + $groupBy = UsageQuery::groupBy('service'); + $groupByInterval = UsageQuery::groupByInterval('time', '1h'); + $regular = Query::equal('metric', ['bandwidth']); + + $this->assertTrue(UsageQuery::isGroupBy($groupBy)); + $this->assertFalse(UsageQuery::isGroupBy($groupByInterval)); + $this->assertFalse(UsageQuery::isGroupBy($regular)); + } + + public function testExtractGroupByReturnsAllMatches(): void + { + $byService = UsageQuery::groupBy('service'); + $byPath = UsageQuery::groupBy('path'); + $interval = UsageQuery::groupByInterval('time', '1h'); + $equal = Query::equal('metric', ['bandwidth']); + + $queries = [$equal, $byService, $interval, $byPath]; + + $extracted = UsageQuery::extractGroupBy($queries); + + $this->assertCount(2, $extracted); + $this->assertEquals('service', $extracted[0]->getAttribute()); + $this->assertEquals('path', $extracted[1]->getAttribute()); + } + + public function testExtractGroupByReturnsEmptyWhenAbsent(): void + { + $queries = [ + Query::equal('metric', ['bandwidth']), + UsageQuery::groupByInterval('time', '1h'), + ]; + + $this->assertSame([], UsageQuery::extractGroupBy($queries)); + } + + public function testRemoveGroupBy(): void + { + $byService = UsageQuery::groupBy('service'); + $byPath = UsageQuery::groupBy('path'); + $interval = UsageQuery::groupByInterval('time', '1h'); + $equal = Query::equal('metric', ['bandwidth']); + + $queries = [$equal, $byService, $interval, $byPath]; + + $remaining = UsageQuery::removeGroupBy($queries); + + $this->assertCount(2, $remaining); + foreach ($remaining as $query) { + $this->assertNotEquals(UsageQuery::TYPE_GROUP_BY, $query->getMethod()); + } + } + + public function testGroupByParseRoundTrip(): void + { + $json = json_encode([ + 'method' => UsageQuery::TYPE_GROUP_BY, + 'attribute' => 'service', + 'values' => [], + ]); + $this->assertIsString($json); + + $parsed = UsageQuery::parse($json); + + $this->assertEquals(UsageQuery::TYPE_GROUP_BY, $parsed->getMethod()); + $this->assertEquals('service', $parsed->getAttribute()); + $this->assertEquals([], $parsed->getValues()); + } + + public function testExtractGroupByFromParsedQuery(): void + { + // Queries created via Query::parse() are base Query objects, not UsageQuery. + $parsedGroupBy = new Query(UsageQuery::TYPE_GROUP_BY, 'service', []); + $equal = Query::equal('metric', ['bandwidth']); + + $extracted = UsageQuery::extractGroupBy([$equal, $parsedGroupBy]); + + $this->assertCount(1, $extracted); + $this->assertEquals('service', $extracted[0]->getAttribute()); + } } From 3d49192c9621aee26db1715cfdb2fad6adb3a1ed Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 27 May 2026 13:22:15 +0000 Subject: [PATCH 17/18] feat(clickhouse): support groupBy dimension columns in aggregated find Extends findAggregatedFromTable so callers can bucket aggregated rows by arbitrary dimension columns (service, path, status, ...) alongside the existing time bucket. - parseQueries() collects groupBy attributes into parsed[groupBy] and validates each against the type-specific column set (Metric::EVENT_COLUMNS / Metric::GAUGE_COLUMNS) via the new validateGroupByAttribute() helper. Unknown columns raise a descriptive Exception listing what is allowed. - findFromTable() rejects groupBy without groupByInterval - keeps the contract simple (caller always supplies a time bucket too) and matches the Database adapter behaviour. - findAggregatedFromTable() appends the escaped dim columns to both SELECT and GROUP BY. parseAggregatedResults() already passes unknown keys through to the Metric so the dim values surface as plain getters (getService, getPath, ...). - Integration tests: single-dim groupBy(service) over 1d aggregates to the expected per-service sums; multi-dim groupBy(service)+groupBy(path) over 1h produces the cartesian grouping with correct sums per pair. --- src/Usage/Adapter/ClickHouse.php | 57 ++++++++++++++++++-- tests/Usage/Adapter/ClickHouseTest.php | 73 ++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 4 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index b198ca1..ad7a33a 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1179,6 +1179,27 @@ private function validateAttributeName(string $attributeName, string $type = 'ev throw new Exception("Invalid attribute name: {$attributeName}"); } + /** + * Validate that a groupBy attribute is an aggregable dimension column. + * + * Restricted to the indexed dimension columns for the table type — `metric`, + * `value` and `time` are excluded since they are already part of the + * aggregation (metric is in the SELECT, time is bucketed via + * groupByInterval, value is the measured quantity). + * + * @throws Exception + */ + private function validateGroupByAttribute(string $attribute, string $type): bool + { + $allowed = $type === Usage::TYPE_GAUGE ? Metric::GAUGE_COLUMNS : Metric::EVENT_COLUMNS; + + if (in_array($attribute, $allowed, true)) { + return true; + } + + throw new Exception("Invalid groupBy attribute '{$attribute}' for {$type}. Allowed: " . implode(', ', $allowed)); + } + /** * Columns available in the events daily (pre-aggregated) table. */ @@ -1567,6 +1588,10 @@ private function findFromTable(array $queries, string $type): array throw new Exception('Cursor pagination cannot be combined with groupByInterval'); } + if (!empty($parsed['groupBy']) && !isset($parsed['groupByInterval'])) { + throw new Exception('groupBy requires groupByInterval to be specified'); + } + // Check if groupByInterval is requested if (isset($parsed['groupByInterval'])) { return $this->findAggregatedFromTable($parsed, $fromTable, $type); @@ -1629,7 +1654,7 @@ private function findFromTable(array $queries, string $type): array * toStartOfInterval(time, INTERVAL 1 HOUR) as time * FROM table WHERE ... GROUP BY metric, time ORDER BY time ASC * - * @param array{filters: array, params: array, orderBy?: array, limit?: int, offset?: int, groupByInterval?: string} $parsed Parsed query data from parseQueries() + * @param array{filters: array, params: array, orderBy?: array, limit?: int, offset?: int, groupByInterval?: string, groupBy?: array} $parsed Parsed query data from parseQueries() * @param string $fromTable Fully qualified table reference * @param string $type 'event' or 'gauge' * @return array @@ -1650,6 +1675,18 @@ private function findAggregatedFromTable(array $parsed, string $fromTable, strin // then alias back to 'time' in outer context for consistent Metric parsing. $timeBucketExpr = "toStartOfInterval(time, {$intervalSql})"; + $groupByDims = $parsed['groupBy'] ?? []; + $dimSelect = ''; + $dimGroup = ''; + if (!empty($groupByDims)) { + $escapedDims = array_map( + fn (string $dim): string => $this->escapeIdentifier($dim), + $groupByDims + ); + $dimSelect = ', ' . implode(', ', $escapedDims); + $dimGroup = ', ' . implode(', ', $escapedDims); + } + $whereData = $this->buildWhereClause($parsed['filters'], $parsed['params']); $whereClause = $whereData['clause']; $params = $whereData['params']; @@ -1676,9 +1713,9 @@ private function findAggregatedFromTable(array $parsed, string $fromTable, strin $offsetClause = isset($parsed['offset']) ? ' OFFSET {offset:UInt64}' : ''; $sql = " - SELECT metric, {$valueExpr}, {$timeBucketExpr} as bucket + SELECT metric, {$valueExpr}, {$timeBucketExpr} as bucket{$dimSelect} FROM {$fromTable}{$whereClause} - GROUP BY metric, bucket{$orderClause}{$limitClause}{$offsetClause} + GROUP BY metric, bucket{$dimGroup}{$orderClause}{$limitClause}{$offsetClause} FORMAT JSON "; @@ -2764,7 +2801,7 @@ private function buildOrderBySql(array $orderAttributes, bool $flip = false): ar * * @param array $queries * @param string $type 'event' or 'gauge' — used for attribute validation - * @return array{filters: array, params: array, orderBy?: array, orderAttributes?: array, limit?: int, offset?: int, groupByInterval?: string, cursor?: array, cursorDirection?: string} + * @return array{filters: array, params: array, orderBy?: array, orderAttributes?: array, limit?: int, offset?: int, groupByInterval?: string, groupBy?: array, cursor?: array, cursorDirection?: string} * @throws Exception */ private function parseQueries(array $queries, string $type = 'event'): array @@ -2776,6 +2813,7 @@ private function parseQueries(array $queries, string $type = 'event'): array $limit = null; $offset = null; $groupByInterval = null; + $groupBy = []; $cursor = null; $cursorDirection = null; $paramCounter = 0; @@ -3011,6 +3049,13 @@ private function parseQueries(array $queries, string $type = 'event'): array } $groupByInterval = $interval; break; + + case UsageQuery::TYPE_GROUP_BY: + $this->validateGroupByAttribute($attribute, $type); + if (!in_array($attribute, $groupBy, true)) { + $groupBy[] = $attribute; + } + break; } } @@ -3036,6 +3081,10 @@ private function parseQueries(array $queries, string $type = 'event'): array $result['groupByInterval'] = $groupByInterval; } + if (!empty($groupBy)) { + $result['groupBy'] = $groupBy; + } + if ($cursor !== null && $cursorDirection !== null) { $result['cursor'] = $cursor; $result['cursorDirection'] = $cursorDirection; diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index b9bd08e..66b5d3d 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -1241,6 +1241,79 @@ public function testCursorWithGroupByIntervalThrows(): void ], Usage::TYPE_EVENT); } + public function testGroupByServiceDailyAggregates(): void + { + $this->usage->purge(); + + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'gb-service', 'value' => 10, 'tags' => ['service' => 'storage']], + ['metric' => 'gb-service', 'value' => 25, 'tags' => ['service' => 'storage']], + ['metric' => 'gb-service', 'value' => 5, 'tags' => ['service' => 'databases']], + ], Usage::TYPE_EVENT)); + + $start = (new \DateTime())->modify('-1 day')->format('Y-m-d\TH:i:s'); + $end = (new \DateTime())->modify('+1 day')->format('Y-m-d\TH:i:s'); + + $results = $this->usage->find([ + UsageQuery::groupByInterval('time', '1d'), + UsageQuery::groupBy('service'), + Query::equal('metric', ['gb-service']), + Query::greaterThanEqual('time', $start), + Query::lessThanEqual('time', $end), + ], Usage::TYPE_EVENT); + + $this->assertGreaterThanOrEqual(2, count($results)); + + $byService = []; + foreach ($results as $row) { + $service = $row->getService(); + $this->assertNotNull($service, 'groupBy(service) should surface the service dimension on each Metric'); + $byService[$service] = ($byService[$service] ?? 0) + $row->getValue(); + } + + $this->assertEquals(35, $byService['storage'] ?? null); + $this->assertEquals(5, $byService['databases'] ?? null); + } + + public function testGroupByMultipleDimensionsHourly(): void + { + $this->usage->purge(); + + $this->assertTrue($this->usage->addBatch([ + ['metric' => 'gb-multi', 'value' => 1, 'tags' => ['service' => 'storage', 'path' => '/v1/a']], + ['metric' => 'gb-multi', 'value' => 2, 'tags' => ['service' => 'storage', 'path' => '/v1/a']], + ['metric' => 'gb-multi', 'value' => 4, 'tags' => ['service' => 'storage', 'path' => '/v1/b']], + ['metric' => 'gb-multi', 'value' => 8, 'tags' => ['service' => 'databases', 'path' => '/v1/a']], + ], Usage::TYPE_EVENT)); + + $start = (new \DateTime())->modify('-1 hour')->format('Y-m-d\TH:i:s'); + $end = (new \DateTime())->modify('+1 hour')->format('Y-m-d\TH:i:s'); + + $results = $this->usage->find([ + UsageQuery::groupByInterval('time', '1h'), + UsageQuery::groupBy('service'), + UsageQuery::groupBy('path'), + Query::equal('metric', ['gb-multi']), + Query::greaterThanEqual('time', $start), + Query::lessThanEqual('time', $end), + ], Usage::TYPE_EVENT); + + $this->assertGreaterThanOrEqual(3, count($results)); + + $byPair = []; + foreach ($results as $row) { + $svc = $row->getService(); + $path = $row->getPath(); + $this->assertNotNull($svc); + $this->assertNotNull($path); + $byPair["{$svc}|{$path}"] = ($byPair["{$svc}|{$path}"] ?? 0) + $row->getValue(); + } + + $this->assertEquals(3, $byPair['storage|/v1/a'] ?? null); + $this->assertEquals(4, $byPair['storage|/v1/b'] ?? null); + $this->assertEquals(8, $byPair['databases|/v1/a'] ?? null); + } + public function testNotEqualQuery(): void { // Fixture: requests x2, bandwidth x1 in events From 29d92922b16a0a0d302267fe7ade103bd2923ac9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 27 May 2026 13:22:23 +0000 Subject: [PATCH 18/18] feat(database-adapter): validate groupBy queries, share contract with ClickHouse The Database adapter does not push aggregation through utopia-php/database (no groupBy on the base Query class), so groupBy queries continue to be elided from the converted query list - callers get raw rows just as they do today with groupByInterval. The change is in validation: rejecting malformed groupBy use up front means the negative-case contract is identical on both backends. - validateGroupByQueries() runs at the top of convertQueriesToDatabase. Rejects groupBy attributes outside the union of EVENT_COLUMNS and GAUGE_COLUMNS (the Database adapter shares one collection for both types). Rejects groupBy without an accompanying groupByInterval. - Negative-case integration tests live in UsageBase so both adapters exercise them: unknown attribute throws, groupBy-without-interval throws. --- src/Usage/Adapter/Database.php | 55 ++++++++++++++++++++++++++++++++-- tests/Usage/UsageBase.php | 23 ++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 700e917..f0b8b29 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -367,9 +367,13 @@ public function sumDaily(array $queries = [], string $attribute = 'value'): int * * @param array $queries * @return array + * @throws \Exception When a groupBy attribute is not a valid dimension column, + * or when groupBy is used without groupByInterval. */ private function convertQueriesToDatabase(array $queries): array { + $this->validateGroupByQueries($queries); + $dbQueries = []; foreach ($queries as $query) { $method = $query->getMethod(); @@ -474,8 +478,10 @@ private function convertQueriesToDatabase(array $queries): array break; case UsageQuery::TYPE_GROUP_BY_INTERVAL: - // groupByInterval is not supported by the Database adapter. - // Silently skip — callers get raw (non-aggregated) results. + case UsageQuery::TYPE_GROUP_BY: + // groupByInterval and groupBy are not pushed down to the + // Database adapter; callers get raw (non-aggregated) results. + // Validation runs in validateGroupByQueries() before this loop. break; } } @@ -483,6 +489,51 @@ private function convertQueriesToDatabase(array $queries): array return $dbQueries; } + /** + * Validate groupBy / groupByInterval interactions in the supplied queries. + * + * Mirrors the ClickHouse adapter contract: groupBy attributes must exist on + * the matching schema (event vs gauge — we default to the broader event set + * for the Database adapter since both share one collection), and groupBy + * must always be paired with groupByInterval so the cloud-facing API stays + * consistent across backends. + * + * @param array $queries + * @throws \Exception + */ + private function validateGroupByQueries(array $queries): void + { + $hasGroupBy = false; + $hasGroupByInterval = false; + $allowed = array_unique(array_merge(Metric::EVENT_COLUMNS, Metric::GAUGE_COLUMNS)); + + foreach ($queries as $query) { + $method = $query->getMethod(); + + if ($method === UsageQuery::TYPE_GROUP_BY_INTERVAL) { + $hasGroupByInterval = true; + continue; + } + + if ($method !== UsageQuery::TYPE_GROUP_BY) { + continue; + } + + $hasGroupBy = true; + $attribute = $query->getAttribute(); + + if (!in_array($attribute, $allowed, true)) { + throw new \Exception( + "Invalid groupBy attribute '{$attribute}'. Allowed: " . implode(', ', $allowed) + ); + } + } + + if ($hasGroupBy && !$hasGroupByInterval) { + throw new \Exception('groupBy requires groupByInterval to be specified'); + } + } + /** * @param array $queries * @param string|null $type diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index e53f0fd..6ac3a37 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -639,4 +639,27 @@ public function testGroupByIntervalWithLimitOffset(): void $this->assertGreaterThanOrEqual(1, count($results)); } + + public function testGroupByUnknownAttributeThrows(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessageMatches("/groupBy attribute 'not_a_column'/"); + + $this->usage->find([ + UsageQuery::groupByInterval('time', '1h'), + UsageQuery::groupBy('not_a_column'), + Query::equal('metric', ['gbi-requests']), + ], Usage::TYPE_EVENT); + } + + public function testGroupByWithoutGroupByIntervalThrows(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessageMatches('/groupBy requires groupByInterval/'); + + $this->usage->find([ + UsageQuery::groupBy('service'), + Query::equal('metric', ['gbi-requests']), + ], Usage::TYPE_EVENT); + } }