diff --git a/lambdas/functions/webhook/src/ConfigLoader.test.ts b/lambdas/functions/webhook/src/ConfigLoader.test.ts index 61ff16ff45..fba2c121c5 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.test.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.test.ts @@ -168,6 +168,54 @@ describe('ConfigLoader Tests', () => { 'Failed to load config: Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', // eslint-disable-line max-len ); }); + + it('should load config successfully from multiple paths', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; + process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; + + const partialMatcher1 = + '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; + const partialMatcher2 = + ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}]'; + + const combinedMatcherConfig = [ + { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['a']], exactMatch: true } }, + { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['b']], exactMatch: true } }, + ]; + + vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { + if (paramPath === '/path/to/matcher/config-1') return partialMatcher1; + if (paramPath === '/path/to/matcher/config-2') return partialMatcher2; + if (paramPath === '/path/to/webhook/secret') return 'secret'; + return ''; + }); + + const config: ConfigWebhook = await ConfigWebhook.load(); + + expect(config.matcherConfig).toEqual(combinedMatcherConfig); + expect(config.webhookSecret).toBe('secret'); + }); + + it('should throw error if config loading fails from multiple paths', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; + process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; + + const partialMatcher1 = + '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; + const partialMatcher2 = + ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}'; + + vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { + if (paramPath === '/path/to/matcher/config-1') return partialMatcher1; + if (paramPath === '/path/to/matcher/config-2') return partialMatcher2; + if (paramPath === '/path/to/webhook/secret') return 'secret'; + return ''; + }); + + await expect(ConfigWebhook.load()).rejects.toThrow( + "Failed to load config: Failed to parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196 (line 1 column 197)", // eslint-disable-line max-len + ); + }); }); describe('ConfigWebhookEventBridge', () => { @@ -229,6 +277,32 @@ describe('ConfigLoader Tests', () => { expect(config.matcherConfig).toEqual(matcherConfig); }); + it('should load config successfully from multiple paths with repo allow list', async () => { + process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; + + const partial1 = + '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["x"]],"exactMatch":true}}'; + const partial2 = + ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["y"]],"exactMatch":true}}]'; + + const combined: RunnerMatcherConfig[] = [ + { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['x']], exactMatch: true } }, + { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['y']], exactMatch: true } }, + ]; + + vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { + if (paramPath === '/path/to/matcher/config-1') return partial1; + if (paramPath === '/path/to/matcher/config-2') return partial2; + return ''; + }); + + const config: ConfigDispatcher = await ConfigDispatcher.load(); + + expect(config.repositoryAllowList).toEqual(['repo1', 'repo2']); + expect(config.matcherConfig).toEqual(combined); + }); + it('should throw error if config loading fails', async () => { vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { throw new Error(`Parameter ${paramPath} not found`); diff --git a/lambdas/functions/webhook/src/ConfigLoader.ts b/lambdas/functions/webhook/src/ConfigLoader.ts index 646acfaa53..63b94db8de 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.ts @@ -87,9 +87,36 @@ abstract class BaseConfig { } } -export class ConfigWebhook extends BaseConfig { - repositoryAllowList: string[] = []; +abstract class MatcherAwareConfig extends BaseConfig { matcherConfig: RunnerMatcherConfig[] = []; + + protected async loadMatcherConfig(paramPathsEnv: string) { + if (!paramPathsEnv || paramPathsEnv === 'undefined' || paramPathsEnv === 'null' || !paramPathsEnv.includes(':')) { + await this.loadParameter(paramPathsEnv, 'matcherConfig'); + return; + } + + const paths = paramPathsEnv + .split(':') + .map((p) => p.trim()) + .filter(Boolean); + let combinedString = ''; + + for (const path of paths) { + await this.loadParameter(path, 'matcherConfig'); + combinedString += this.matcherConfig; + } + + try { + this.matcherConfig = JSON.parse(combinedString); + } catch (error) { + this.configLoadingErrors.push(`Failed to parse combined matcher config: ${(error as Error).message}`); + } + } +} + +export class ConfigWebhook extends MatcherAwareConfig { + repositoryAllowList: string[] = []; webhookSecret: string = ''; workflowJobEventSecondaryQueue: string = ''; @@ -97,7 +124,7 @@ export class ConfigWebhook extends BaseConfig { this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []); await Promise.all([ - this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig'), + this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH), this.loadParameter(process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET, 'webhookSecret'), ]); @@ -121,14 +148,13 @@ export class ConfigWebhookEventBridge extends BaseConfig { } } -export class ConfigDispatcher extends BaseConfig { +export class ConfigDispatcher extends MatcherAwareConfig { repositoryAllowList: string[] = []; - matcherConfig: RunnerMatcherConfig[] = []; workflowJobEventSecondaryQueue: string = ''; // Deprecated async loadConfig(): Promise { this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []); - await this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig'); + await this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH); validateRunnerMatcherConfig(this); } diff --git a/modules/webhook/direct/README.md b/modules/webhook/direct/README.md index fa8848b8ec..ee3db410b9 100644 --- a/modules/webhook/direct/README.md +++ b/modules/webhook/direct/README.md @@ -40,7 +40,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list = optional(list(string), [])
    kms_key_arn           = optional(string, null)
    log_level             = optional(string, "info")
    lambda_runtime        = optional(string, "nodejs22.x")
    aws_partition         = optional(string, "aws")
    lambda_architecture   = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = object({
      name    = string
      arn     = string
      version = string
    })
  })
| n/a | yes | +| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list = optional(list(string), [])
    kms_key_arn           = optional(string, null)
    log_level             = optional(string, "info")
    lambda_runtime        = optional(string, "nodejs22.x")
    aws_partition         = optional(string, "aws")
    lambda_architecture   = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = list(object({
      name    = string
      arn     = string
      version = string
    }))
  })
| n/a | yes | ## Outputs diff --git a/modules/webhook/direct/variables.tf b/modules/webhook/direct/variables.tf index 4acf670f38..2a1b559c92 100644 --- a/modules/webhook/direct/variables.tf +++ b/modules/webhook/direct/variables.tf @@ -41,10 +41,10 @@ variable "config" { }), {}) lambda_tags = optional(map(string), {}) api_gw_source_arn = string - ssm_parameter_runner_matcher_config = object({ + ssm_parameter_runner_matcher_config = list(object({ name = string arn = string version = string - }) + })) }) } diff --git a/modules/webhook/direct/webhook.tf b/modules/webhook/direct/webhook.tf index 362ed3e044..fda61dfa91 100644 --- a/modules/webhook/direct/webhook.tf +++ b/modules/webhook/direct/webhook.tf @@ -26,8 +26,8 @@ resource "aws_lambda_function" "webhook" { POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.tracing_config.capture_error PARAMETER_GITHUB_APP_WEBHOOK_SECRET = var.config.github_app_parameters.webhook_secret.name REPOSITORY_ALLOW_LIST = jsonencode(var.config.repository_white_list) - PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.ssm_parameter_runner_matcher_config.name - PARAMETER_RUNNER_MATCHER_VERSION = var.config.ssm_parameter_runner_matcher_config.version # enforce cold start after Changes in SSM parameter + PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name]) + PARAMETER_RUNNER_MATCHER_VERSION = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) # enforce cold start after Changes in SSM parameter } : k => v if v != null } } @@ -134,7 +134,12 @@ resource "aws_iam_role_policy" "webhook_ssm" { role = aws_iam_role.webhook_lambda.name policy = templatefile("${path.module}/../policies/lambda-ssm.json", { - resource_arns = jsonencode([var.config.github_app_parameters.webhook_secret.arn, var.config.ssm_parameter_runner_matcher_config.arn]) + resource_arns = jsonencode( + concat( + [var.config.github_app_parameters.webhook_secret.arn], + [for p in var.config.ssm_parameter_runner_matcher_config : p.arn] + ) + ) }) } diff --git a/modules/webhook/eventbridge/README.md b/modules/webhook/eventbridge/README.md index b02fe72f60..329ac3c232 100644 --- a/modules/webhook/eventbridge/README.md +++ b/modules/webhook/eventbridge/README.md @@ -54,7 +54,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list = optional(list(string), [])
    kms_key_arn           = optional(string, null)
    log_level             = optional(string, "info")
    lambda_runtime        = optional(string, "nodejs22.x")
    aws_partition         = optional(string, "aws")
    lambda_architecture   = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = object({
      name    = string
      arn     = string
      version = string
    })
    accept_events = optional(list(string), null)
  })
| n/a | yes | +| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list = optional(list(string), [])
    kms_key_arn           = optional(string, null)
    log_level             = optional(string, "info")
    lambda_runtime        = optional(string, "nodejs22.x")
    aws_partition         = optional(string, "aws")
    lambda_architecture   = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = list(object({
      name    = string
      arn     = string
      version = string
    }))
    accept_events = optional(list(string), null)
  })
| n/a | yes | ## Outputs diff --git a/modules/webhook/eventbridge/dispatcher.tf b/modules/webhook/eventbridge/dispatcher.tf index 2e311c533f..1ddc961a6f 100644 --- a/modules/webhook/eventbridge/dispatcher.tf +++ b/modules/webhook/eventbridge/dispatcher.tf @@ -44,8 +44,8 @@ resource "aws_lambda_function" "dispatcher" { POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.tracing_config.capture_http_requests POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.tracing_config.capture_error # Parameters required for lambda configuration - PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.ssm_parameter_runner_matcher_config.name - PARAMETER_RUNNER_MATCHER_VERSION = var.config.ssm_parameter_runner_matcher_config.version # enforce cold start after Changes in SSM parameter + PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name]) + PARAMETER_RUNNER_MATCHER_VERSION = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) # enforce cold start after Changes in SSM parameter REPOSITORY_ALLOW_LIST = jsonencode(var.config.repository_white_list) } : k => v if v != null } @@ -129,7 +129,12 @@ resource "aws_iam_role_policy" "dispatcher_ssm" { role = aws_iam_role.dispatcher_lambda.name policy = templatefile("${path.module}/../policies/lambda-ssm.json", { - resource_arns = jsonencode([var.config.ssm_parameter_runner_matcher_config.arn]) + resource_arns = jsonencode( + concat( + [var.config.github_app_parameters.webhook_secret.arn], + [for p in var.config.ssm_parameter_runner_matcher_config : p.arn] + ) + ) }) } diff --git a/modules/webhook/eventbridge/variables.tf b/modules/webhook/eventbridge/variables.tf index 18eaeb78dd..8a884a6ba3 100644 --- a/modules/webhook/eventbridge/variables.tf +++ b/modules/webhook/eventbridge/variables.tf @@ -41,11 +41,11 @@ variable "config" { }), {}) lambda_tags = optional(map(string), {}) api_gw_source_arn = string - ssm_parameter_runner_matcher_config = object({ + ssm_parameter_runner_matcher_config = list(object({ name = string arn = string version = string - }) + })) accept_events = optional(list(string), null) }) } diff --git a/modules/webhook/eventbridge/webhook.tf b/modules/webhook/eventbridge/webhook.tf index c57b6da5e3..8792403134 100644 --- a/modules/webhook/eventbridge/webhook.tf +++ b/modules/webhook/eventbridge/webhook.tf @@ -31,7 +31,7 @@ resource "aws_lambda_function" "webhook" { ACCEPT_EVENTS = jsonencode(var.config.accept_events) EVENT_BUS_NAME = aws_cloudwatch_event_bus.main.name PARAMETER_GITHUB_APP_WEBHOOK_SECRET = var.config.github_app_parameters.webhook_secret.name - PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.ssm_parameter_runner_matcher_config.name + PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name]) } : k => v if v != null } } diff --git a/modules/webhook/webhook.tf b/modules/webhook/webhook.tf index 5f425bcef2..16b526415b 100644 --- a/modules/webhook/webhook.tf +++ b/modules/webhook/webhook.tf @@ -4,12 +4,39 @@ locals { # sorted list runner_matcher_config_sorted = [for k in sort(keys(local.runner_matcher_config)) : local.runner_matcher_config[k]] + + # Define worst-case dummy ARN/ID lengths + worst_case_arn = join("", [for i in range(0, 127) : "X"]) # ARN length assuming 80-char queue name, longest partition & region + worst_case_id = join("", [for i in range(0, 135) : "Y"]) # SQS URL length for same worst-case scenario + + # Compute worst-case JSON length + worst_case_json_length = length(jsonencode([for r in local.runner_matcher_config_sorted : merge(r, { arn = local.worst_case_arn, id = local.worst_case_id })])) + + # Set max chunk size based on SSM tier + # AWS SSM limits: + # - Standard: 4096 bytes + # - Advanced: 8192 bytes + # We leave a small safety margin to avoid hitting the exact limit + # (e.g., escaped characters or minor overhead could exceed the limit) + max_chunk_size = var.matcher_config_parameter_store_tier == "Advanced" ? 8000 : 4000 + + # Calculate total number of chunks + total_chunks = ceil(local.worst_case_json_length / local.max_chunk_size) + + # Encode the sorted matcher config as JSON + matcher_json = jsonencode(local.runner_matcher_config_sorted) + chunk_size = ceil(length(local.matcher_json) / local.total_chunks) + + # Split JSON into chunks safely under the SSM limit + matcher_json_chunks = [for i in range(0, length(local.matcher_json), local.chunk_size) : substr(local.matcher_json, i, local.chunk_size)] } resource "aws_ssm_parameter" "runner_matcher_config" { - name = "${var.ssm_paths.root}/${var.ssm_paths.webhook}/runner-matcher-config" + count = local.total_chunks + + name = "${var.ssm_paths.root}/${var.ssm_paths.webhook}/runner-matcher-config${local.total_chunks > 1 ? "-${count.index}" : ""}" type = "String" - value = jsonencode(local.runner_matcher_config_sorted) + value = local.matcher_json_chunks[count.index] tier = var.matcher_config_parameter_store_tier } @@ -46,7 +73,13 @@ module "direct" { lambda_tags = var.lambda_tags, matcher_config_parameter_store_tier = var.matcher_config_parameter_store_tier, api_gw_source_arn = "${aws_apigatewayv2_api.webhook.execution_arn}/*/*/${local.webhook_endpoint}" - ssm_parameter_runner_matcher_config = aws_ssm_parameter.runner_matcher_config + ssm_parameter_runner_matcher_config = [ + for p in aws_ssm_parameter.runner_matcher_config : { + name = p.name + arn = p.arn + version = p.version + } + ] } } @@ -81,8 +114,14 @@ module "eventbridge" { tracing_config = var.tracing_config, lambda_tags = var.lambda_tags, api_gw_source_arn = "${aws_apigatewayv2_api.webhook.execution_arn}/*/*/${local.webhook_endpoint}" - ssm_parameter_runner_matcher_config = aws_ssm_parameter.runner_matcher_config - accept_events = var.eventbridge.accept_events + ssm_parameter_runner_matcher_config = [ + for p in aws_ssm_parameter.runner_matcher_config : { + name = p.name + arn = p.arn + version = p.version + } + ] + accept_events = var.eventbridge.accept_events } }