Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(kinesisfirehose): add HTTP Endpoint and Datadog destination #33657

Open
wants to merge 20 commits into
base: main
Choose a base branch
from
Open
Prev Previous commit
Next Next commit
fix: build
  • Loading branch information
Benjamin Pottier authored and benjaminpottier committed Mar 5, 2025
commit 134e93f0fe07bfed5cc9a34cd412a2c9e32383bc
11 changes: 11 additions & 0 deletions packages/aws-cdk-lib/aws-kinesisfirehose/README.md
Original file line number Diff line number Diff line change
@@ -85,6 +85,17 @@ Amazon Data Firehose supports multiple AWS and third-party services as destinati
Currently in the AWS CDK, only S3 is implemented as an L2 construct destination. Other destinations can still be configured using L1 constructs. See [kinesisfirehose-destinations](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-kinesisfirehose-destinations-readme.html)
for the implementations of these destinations.

### HTTP

Defining a delivery stream with an HTTP destination:

```ts
declare const endpointConfig: firehouse.HTTPEndpointConifg;
const httpDestination = new firehouse.HTTPEndpoint({
endpointConfig,
});
```

### Datadog

Defining a delivery stream with a Datadog destination:
10 changes: 6 additions & 4 deletions packages/aws-cdk-lib/aws-kinesisfirehose/lib/datadog.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Attribute, HTTPBackupMode, HTTPEndpoint } from './http-endpoint';
import { HTTPAttribute, HTTPBackupMode, HTTPEndpoint } from './http-endpoint';
import { ISecret } from '../../aws-secretsmanager';
import { Duration, Size } from '../../core';

@@ -100,9 +100,10 @@ export interface DatadogProps {
* Datadog tags to apply for filtering.
* @default - No tags.
*/
readonly tags?: Attribute[];
readonly tags?: HTTPAttribute[];
/**
* @default - failed only
* Describes the S3 bucket backup options for the data that Kinesis Data Firehose delivers to Datadog.
* @default - Failed only
*/
readonly backupMode?: HTTPBackupMode;
}
@@ -113,7 +114,7 @@ export interface DatadogProps {
export class Datadog extends HTTPEndpoint {
constructor(props: DatadogProps) {
super({
endpoint: {
endpointConfig: {
url: props.url,
secret: props.apiKey,
},
@@ -124,6 +125,7 @@ export class Datadog extends HTTPEndpoint {
retryOptions: {
duration: Duration.seconds(60),
},
backupMode: props.backupMode ?? HTTPBackupMode.FAILED,
attributes: props.tags ?? [],
});
}
87 changes: 47 additions & 40 deletions packages/aws-cdk-lib/aws-kinesisfirehose/lib/http-endpoint.ts
Original file line number Diff line number Diff line change
@@ -7,37 +7,37 @@ import { ISecret } from '../../aws-secretsmanager';
import { Duration, Size } from '../../core';

/**
*
* Kinesis Data Firehose uses the content encoding to compress the body of a request before sending the request to the destination.
*/
export enum HTTPCompression {
/**
*
* GZIP
*/
GZIP = 'GZIP',
/**
*
* NONE
*/
NONE = 'NONE',
}

/**
*
* Describes the S3 bucket backup options for the data that Kinesis Data Firehose delivers to the HTTP endpoint destination.
*/
export enum HTTPBackupMode {
/**
*
* Back up only the documents that Kinesis Data Firehose could not deliver.
*/
FAILED = 'FailedDataOnly',
/**
*
* Back up all documents.
*/
ALL = 'AllData',
}

/**
* The buffering options that can be used before data is delivered to the specified destination.
*/
export interface BufferingHints {
export interface HTTPBufferingHints {
/**
* The higher interval allows more time to collect data and the size of data may be bigger. The lower interval sends the data more frequently and may be more advantageous when looking at shorter cycles of data activity.
* @default 60 seconds
@@ -51,84 +51,91 @@ export interface BufferingHints {
}

/**
*
* Describes the retry behavior in case Kinesis Data Firehose is unable to deliver data to the specified HTTP endpoint destination, or if it doesn't receive a valid acknowledgment of receipt from the specified HTTP endpoint destination.
*/
export interface RetryOptions {
export interface HTTPRetryOptions {
/**
*
* The total amount of time that Kinesis Data Firehose spends on retries.
*/
readonly duration: Duration;
}

/**
*
* Describes the configuration of the HTTP endpoint to which Kinesis Firehose delivers data.
*/
export interface Endpoint {
export interface HTTPEndpointConfig {
/**
*
* The URL of the HTTP endpoint selected as the destination.
*/
readonly url: string;
/**
* @default - Not used
* The access key required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination.
* @default - None
*/
readonly accessKey?: string;
/**
*
* @default - Not used
* The secret required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination.
* @default - None
*/
readonly secret?: ISecret;
/**
* @default - Not used
* The name of the HTTP endpoint selected as the destination.
* @default - None
*/
readonly name?: string;
}

/**
*
* Describes the metadata sent to the HTTP endpoint destination.
*/
export interface Attribute {
export interface HTTPAttribute {
/**
*
* The name of the HTTP endpoint common attribute.
*/
readonly name: string;
/**
*
* The value of the HTTP endpoint common attribute.
*/
readonly value: string;
}

/**
*
* Props for defining an HTTP destination of a Kinesis Data Firehose delivery stream.
*/
export interface HTTPEndpointProps extends CommonDestinationProps {
/**
*
* Describes the configuration of the HTTP endpoint to which Kinesis Firehose delivers data.
*/
readonly endpoint: Endpoint;
readonly endpointConfig: HTTPEndpointConfig;
/**
* @default - FailedDataOnly
* Describes the S3 bucket backup options for the data that Kinesis Data Firehose delivers to the HTTP endpoint destination.
* @default - Failed data only
*/
readonly backupMode?: HTTPBackupMode;
/**
* @default - GZIP
* Compress the body of a request before sending the request to the destination.
* @default - None
*/
readonly requestCompression?: HTTPCompression;
/**
* @default -
* The buffering options that can be used before data is delivered to the specified destination.
* @default - None
*/
readonly bufferingHints?: BufferingHints;
readonly bufferingHints?: HTTPBufferingHints;
/**
* @default -
* The total amount of time that Kinesis Data Firehose spends on retries
* @default - None
*/
readonly retryOptions?: RetryOptions;
readonly retryOptions?: HTTPRetryOptions;
/**
* Describes the metadata sent to the HTTP endpoint destination.
* @default - None
*/
readonly attributes?: Attribute[];
readonly attributes?: HTTPAttribute[];
}

/**
*
* An HTTP destination for data from a Kinesis Data Firehose delivery stream.
*/
export class HTTPEndpoint implements IDestination {
constructor(private readonly props: HTTPEndpointProps) {}
@@ -147,16 +154,16 @@ export class HTTPEndpoint implements IDestination {
mode: S3BackupMode.FAILED,
})!; // Probably not a good idea?

if (this.props.endpoint.secret) {
this.props.endpoint.secret.grantRead(role);
if (this.props.endpointConfig.secret) {
this.props.endpointConfig.secret.grantRead(role);
}

return {
httpEndpointDestinationConfiguration: {
endpointConfiguration: {
url: this.props.endpoint.url,
...this.props.endpoint.accessKey && { accessKey: this.props.endpoint.accessKey },
...this.props.endpoint.name && { name: this.props.endpoint.name },
url: this.props.endpointConfig.url,
...this.props.endpointConfig.accessKey && { accessKey: this.props.endpointConfig.accessKey },
...this.props.endpointConfig.name && { name: this.props.endpointConfig.name },
},
...this.props.retryOptions && {
retryOptions: {
@@ -170,7 +177,7 @@ export class HTTPEndpoint implements IDestination {
},
},
requestConfiguration: {
contentEncoding: this.props.requestCompression ?? HTTPCompression.GZIP,
contentEncoding: this.props.requestCompression ?? HTTPCompression.NONE,
...this.props.attributes && {
commonAttributes: [...this.props.attributes.map(attr => ({ attributeName: attr.name, attributeValue: attr.value }))],
},
@@ -180,9 +187,9 @@ export class HTTPEndpoint implements IDestination {
roleArn: role.roleArn,
s3BackupMode: this.props.backupMode ?? HTTPBackupMode.FAILED,
s3Configuration: backupConfig,
...this.props.endpoint.secret && {
...this.props.endpointConfig.secret && {
secretsManagerConfiguration: {
secretArn: this.props.endpoint.secret.secretArn,
secretArn: this.props.endpointConfig.secret.secretArn,
enabled: true,
roleArn: role.roleArn,
},