Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(kinesisfirehose): add HTTP Endpoint and Datadog destination #33657

Open
wants to merge 20 commits into
base: main
Choose a base branch
from
Open
Prev Previous commit
Next Next commit
fix: build errors
  • Loading branch information
Benjamin Pottier authored and benjaminpottier committed Mar 5, 2025
commit 83c7dfa39942a4e1238e10d7618efc34b6a6997f
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/usr/bin/env node
import * as firehose from 'aws-cdk-lib/aws-kinesisfirehose';
import * as cdk from 'aws-cdk-lib';
import { IntegTest } from '@aws-cdk/integ-tests-alpha';
import { Secret } from 'aws-cdk-lib/aws-secretsmanager';

const app = new cdk.App();

const stack = new cdk.Stack(app, 'aws-cdk-firehose-delivery-stream-s3-all-properties');

const deliveryStream = new firehose.DeliveryStream(stack, 'DeliveryStream', {
destination: new firehose.Datadog({
apiKey: Secret.fromSecretNameV2(stack, 'DatadogApiKey', 'DatadogApiKey'),
url: firehose.DatadogLogsEndpointUrl.DATADOG_LOGS_US1,
}),
});

const testCase = new IntegTest(app, 'integ-tests', {
testCases: [stack],
regions: ['us-east-1'],
});

testCase.assertions.awsApiCall('Firehose', 'putRecord', {
DeliveryStreamName: deliveryStream.deliveryStreamName,
Record: {
Data: 'testData123',
},
});
68 changes: 64 additions & 4 deletions packages/aws-cdk-lib/aws-kinesisfirehose/lib/datadog.ts
Original file line number Diff line number Diff line change
@@ -4,45 +4,105 @@ import { DestinationBindOptions, DestinationConfig, IDestination } from './desti
import * as iam from '../../aws-iam';
import { ISecret } from '../../aws-secretsmanager';
import { Duration, Size } from '../../core';
import { createBackupConfig, createBufferingHints, createEncryptionConfig, createLoggingOptions, createProcessingConfig } from './private/helpers';
import { createBackupConfig, createLoggingOptions, createProcessingConfig } from './private/helpers';
import { Bucket } from '../../aws-s3';

/**
* Datadog logs HTTP endpoint URLs
*/
export enum DatadogLogsEndpointUrl {
/**
* US Location
*/
DATADOG_LOGS_US1 = 'https://aws-kinesis-http-intake.logs.datadoghq.com/v1/input',
/**
* US Location
*/
DATADOG_LOGS_US3 = 'https://aws-kinesis-http-intake.logs.us3.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose',
/**
* US Location
*/
DATADOG_LOGS_US5 = 'https://aws-kinesis-http-intake.logs.us5.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose',
/**
* Japan Location
*/
DATADOG_LOGS_AP1 = 'https://aws-kinesis-http-intake.loqs.ap1.datadoqhq.com/api/v2/loqs?dd-protocol=aws-kinesis-firehose',
/**
* EU Location
*/
DATADOG_LOGS_EU = 'https://aws-kinesis-http-intake.logs.datadoghq.eu/v1/input',
/**
* US Government Location
*/
DATADOG_LOGS_GOV = 'https://aws-kinesis-http-intake.logs.ddog-gov.com/v1/input',
}

/**
* Datadog metrics HTTP endpoint URLs
*/
export enum DatadogMetricsEndpointUrl {
/**
* US Location
*/
DATADOG_METRICS_US = 'https://awsmetrics-intake.datadoghq.com/v1/input',
/**
* US Location
*/
DATADOG_METRICS_US5 = 'https://event-platform-intake.us5.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose',
/**
* Japan Location
*/
DATADOG_METRICS_AP1 = 'https://event-platform-intake.ap1.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose',
/**
* EU Location
*/
DATADOG_METRICS_EU = 'https://awsmetrics-intake.datadoghq.eu/v1/input',
}

/**
* Datadog configurations HTTP endpoint URLs
*/
export enum DatadogConfigurationsEndpointUrl {
/**
* US Location
*/
DATADOG_CONFIGURATION_US1 = 'https://cloudplatform-intake.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose',
/**
* US Location
*/
DATADOG_CONFIGURATION_US3 = 'https://cloudplatform-intake.us3.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose',
/**
* US Location
*/
DATADOG_CONFIGURATION_US5 = 'https://cloudplatform-intake.us5.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose',
/**
* Japan Location
*/
DATADOG_CONFIGURATION_AP1 = 'https://cloudplatform-intake.ap1.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose',
/**
* EU Location
*/
DATADOG_CONFIGURATION_EU = 'https://cloudplatform-intake.datadoghq.eu/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose',
/**
* US Government Location
*/
DATADOG_CONFIGURATION_US_GOV = 'https://cloudplatform-intake.ddog-gov.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose',
}

/**
* The buffering options that can be used before data is delivered to the specified destination.
*/
export interface BufferHints {
/**
* The higher interval allows more time to collect data and the size of data may be bigger. The lower interval sends the data more frequently and may be more advantageous when looking at shorter cycles of data activity.
* @default 60 seconds
*/
interval?: Duration;
readonly interval?: Duration;
/**
* The higher buffer size may be lower in cost with higher latency. The lower buffer size will be faster in delivery with higher cost and less latency.
* @default 4 MiB
*/
size?: Size;
readonly size?: Size;
}

/**
@@ -75,7 +135,7 @@ export interface DatadogProps extends CommonDestinationProps {
export class Datadog implements IDestination {
constructor(private readonly props: DatadogProps) { }

bind(scope: Construct, options: DestinationBindOptions): DestinationConfig {
bind(scope: Construct, _options: DestinationBindOptions): DestinationConfig {
const role = this.props.role ?? new iam.Role(scope, 'Datadog Destination Role', {
assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'),
});
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ export interface DestinationConfig {
readonly extendedS3DestinationConfiguration?: CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty;
/**
* Datadog destination configuration properties.
*
*
* @default - Datadog destination is not used.
*/
readonly httpEndpointDestinationConfiguration?: CfnDeliveryStream.HttpEndpointDestinationConfigurationProperty;