/
types.go
2800 lines (2171 loc) · 108 KB
/
types.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Code generated by smithy-go-codegen DO NOT EDIT.
package types
import (
smithydocument "github.com/aws/smithy-go/document"
"time"
)
// Describes the buffering to perform before delivering data to the Serverless
// offering for Amazon OpenSearch Service destination.
type AmazonOpenSearchServerlessBufferingHints struct {
// Buffer incoming data for the specified period of time, in seconds, before
// delivering it to the destination. The default value is 300 (5 minutes).
IntervalInSeconds *int32
// Buffer incoming data to the specified size, in MBs, before delivering it to the
// destination. The default value is 5. We recommend setting this parameter to a
// value greater than the amount of data you typically ingest into the delivery
// stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec, the
// value should be 10 MB or higher.
SizeInMBs *int32
noSmithyDocumentSerde
}
// Describes the configuration of a destination in the Serverless offering for
// Amazon OpenSearch Service.
type AmazonOpenSearchServerlessDestinationConfiguration struct {
// The Serverless offering for Amazon OpenSearch Service index name.
//
// This member is required.
IndexName *string
// The Amazon Resource Name (ARN) of the IAM role to be assumed by Firehose for
// calling the Serverless offering for Amazon OpenSearch Service Configuration API
// and for indexing documents.
//
// This member is required.
RoleARN *string
// Describes the configuration of a destination in Amazon S3.
//
// This member is required.
S3Configuration *S3DestinationConfiguration
// The buffering options. If no value is specified, the default values for
// AmazonopensearchserviceBufferingHints are used.
BufferingHints *AmazonOpenSearchServerlessBufferingHints
// Describes the Amazon CloudWatch logging options for your delivery stream.
CloudWatchLoggingOptions *CloudWatchLoggingOptions
// The endpoint to use when communicating with the collection in the Serverless
// offering for Amazon OpenSearch Service.
CollectionEndpoint *string
// Describes a data processing configuration.
ProcessingConfiguration *ProcessingConfiguration
// The retry behavior in case Firehose is unable to deliver documents to the
// Serverless offering for Amazon OpenSearch Service. The default value is 300 (5
// minutes).
RetryOptions *AmazonOpenSearchServerlessRetryOptions
// Defines how documents should be delivered to Amazon S3. When it is set to
// FailedDocumentsOnly, Firehose writes any documents that could not be indexed to
// the configured Amazon S3 destination, with AmazonOpenSearchService-failed/
// appended to the key prefix. When set to AllDocuments, Firehose delivers all
// incoming records to Amazon S3, and also writes failed documents with
// AmazonOpenSearchService-failed/ appended to the prefix.
S3BackupMode AmazonOpenSearchServerlessS3BackupMode
// The details of the VPC of the Amazon OpenSearch or Amazon OpenSearch Serverless
// destination.
VpcConfiguration *VpcConfiguration
noSmithyDocumentSerde
}
// The destination description in the Serverless offering for Amazon OpenSearch
// Service.
type AmazonOpenSearchServerlessDestinationDescription struct {
// The buffering options.
BufferingHints *AmazonOpenSearchServerlessBufferingHints
// Describes the Amazon CloudWatch logging options for your delivery stream.
CloudWatchLoggingOptions *CloudWatchLoggingOptions
// The endpoint to use when communicating with the collection in the Serverless
// offering for Amazon OpenSearch Service.
CollectionEndpoint *string
// The Serverless offering for Amazon OpenSearch Service index name.
IndexName *string
// Describes a data processing configuration.
ProcessingConfiguration *ProcessingConfiguration
// The Serverless offering for Amazon OpenSearch Service retry options.
RetryOptions *AmazonOpenSearchServerlessRetryOptions
// The Amazon Resource Name (ARN) of the Amazon Web Services credentials.
RoleARN *string
// The Amazon S3 backup mode.
S3BackupMode AmazonOpenSearchServerlessS3BackupMode
// Describes a destination in Amazon S3.
S3DestinationDescription *S3DestinationDescription
// The details of the VPC of the Amazon ES destination.
VpcConfigurationDescription *VpcConfigurationDescription
noSmithyDocumentSerde
}
// Describes an update for a destination in the Serverless offering for Amazon
// OpenSearch Service.
type AmazonOpenSearchServerlessDestinationUpdate struct {
// The buffering options. If no value is specified, AmazonopensearchBufferingHints
// object default values are used.
BufferingHints *AmazonOpenSearchServerlessBufferingHints
// Describes the Amazon CloudWatch logging options for your delivery stream.
CloudWatchLoggingOptions *CloudWatchLoggingOptions
// The endpoint to use when communicating with the collection in the Serverless
// offering for Amazon OpenSearch Service.
CollectionEndpoint *string
// The Serverless offering for Amazon OpenSearch Service index name.
IndexName *string
// Describes a data processing configuration.
ProcessingConfiguration *ProcessingConfiguration
// The retry behavior in case Firehose is unable to deliver documents to the
// Serverless offering for Amazon OpenSearch Service. The default value is 300 (5
// minutes).
RetryOptions *AmazonOpenSearchServerlessRetryOptions
// The Amazon Resource Name (ARN) of the IAM role to be assumed by Firehose for
// calling the Serverless offering for Amazon OpenSearch Service Configuration API
// and for indexing documents.
RoleARN *string
// Describes an update for a destination in Amazon S3.
S3Update *S3DestinationUpdate
noSmithyDocumentSerde
}
// Configures retry behavior in case Firehose is unable to deliver documents to
// the Serverless offering for Amazon OpenSearch Service.
type AmazonOpenSearchServerlessRetryOptions struct {
// After an initial failure to deliver to the Serverless offering for Amazon
// OpenSearch Service, the total amount of time during which Firehose retries
// delivery (including the first attempt). After this time has elapsed, the failed
// documents are written to Amazon S3. Default value is 300 seconds (5 minutes). A
// value of 0 (zero) results in no retries.
DurationInSeconds *int32
noSmithyDocumentSerde
}
// Describes the buffering to perform before delivering data to the Amazon
// OpenSearch Service destination.
type AmazonopensearchserviceBufferingHints struct {
// Buffer incoming data for the specified period of time, in seconds, before
// delivering it to the destination. The default value is 300 (5 minutes).
IntervalInSeconds *int32
// Buffer incoming data to the specified size, in MBs, before delivering it to the
// destination. The default value is 5. We recommend setting this parameter to a
// value greater than the amount of data you typically ingest into the delivery
// stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec, the
// value should be 10 MB or higher.
SizeInMBs *int32
noSmithyDocumentSerde
}
// Describes the configuration of a destination in Amazon OpenSearch Service
type AmazonopensearchserviceDestinationConfiguration struct {
// The ElasticsearAmazon OpenSearch Service index name.
//
// This member is required.
IndexName *string
// The Amazon Resource Name (ARN) of the IAM role to be assumed by Firehose for
// calling the Amazon OpenSearch Service Configuration API and for indexing
// documents.
//
// This member is required.
RoleARN *string
// Describes the configuration of a destination in Amazon S3.
//
// This member is required.
S3Configuration *S3DestinationConfiguration
// The buffering options. If no value is specified, the default values for
// AmazonopensearchserviceBufferingHints are used.
BufferingHints *AmazonopensearchserviceBufferingHints
// Describes the Amazon CloudWatch logging options for your delivery stream.
CloudWatchLoggingOptions *CloudWatchLoggingOptions
// The endpoint to use when communicating with the cluster. Specify either this
// ClusterEndpoint or the DomainARN field.
ClusterEndpoint *string
// Indicates the method for setting up document ID. The supported methods are
// Firehose generated document ID and OpenSearch Service generated document ID.
DocumentIdOptions *DocumentIdOptions
// The ARN of the Amazon OpenSearch Service domain. The IAM role must have
// permissions for DescribeElasticsearchDomain, DescribeElasticsearchDomains, and
// DescribeElasticsearchDomainConfig after assuming the role specified in RoleARN.
DomainARN *string
// The Amazon OpenSearch Service index rotation period. Index rotation appends a
// timestamp to the IndexName to facilitate the expiration of old data.
IndexRotationPeriod AmazonopensearchserviceIndexRotationPeriod
// Describes a data processing configuration.
ProcessingConfiguration *ProcessingConfiguration
// The retry behavior in case Firehose is unable to deliver documents to Amazon
// OpenSearch Service. The default value is 300 (5 minutes).
RetryOptions *AmazonopensearchserviceRetryOptions
// Defines how documents should be delivered to Amazon S3. When it is set to
// FailedDocumentsOnly, Firehose writes any documents that could not be indexed to
// the configured Amazon S3 destination, with AmazonOpenSearchService-failed/
// appended to the key prefix. When set to AllDocuments, Firehose delivers all
// incoming records to Amazon S3, and also writes failed documents with
// AmazonOpenSearchService-failed/ appended to the prefix.
S3BackupMode AmazonopensearchserviceS3BackupMode
// The Amazon OpenSearch Service type name. For Elasticsearch 6.x, there can be
// only one type per index. If you try to specify a new type for an existing index
// that already has another type, Firehose returns an error during run time.
TypeName *string
// The details of the VPC of the Amazon OpenSearch or Amazon OpenSearch Serverless
// destination.
VpcConfiguration *VpcConfiguration
noSmithyDocumentSerde
}
// The destination description in Amazon OpenSearch Service.
type AmazonopensearchserviceDestinationDescription struct {
// The buffering options.
BufferingHints *AmazonopensearchserviceBufferingHints
// Describes the Amazon CloudWatch logging options for your delivery stream.
CloudWatchLoggingOptions *CloudWatchLoggingOptions
// The endpoint to use when communicating with the cluster. Firehose uses either
// this ClusterEndpoint or the DomainARN field to send data to Amazon OpenSearch
// Service.
ClusterEndpoint *string
// Indicates the method for setting up document ID. The supported methods are
// Firehose generated document ID and OpenSearch Service generated document ID.
DocumentIdOptions *DocumentIdOptions
// The ARN of the Amazon OpenSearch Service domain.
DomainARN *string
// The Amazon OpenSearch Service index name.
IndexName *string
// The Amazon OpenSearch Service index rotation period
IndexRotationPeriod AmazonopensearchserviceIndexRotationPeriod
// Describes a data processing configuration.
ProcessingConfiguration *ProcessingConfiguration
// The Amazon OpenSearch Service retry options.
RetryOptions *AmazonopensearchserviceRetryOptions
// The Amazon Resource Name (ARN) of the Amazon Web Services credentials.
RoleARN *string
// The Amazon S3 backup mode.
S3BackupMode AmazonopensearchserviceS3BackupMode
// Describes a destination in Amazon S3.
S3DestinationDescription *S3DestinationDescription
// The Amazon OpenSearch Service type name. This applies to Elasticsearch 6.x and
// lower versions. For Elasticsearch 7.x and OpenSearch Service 1.x, there's no
// value for TypeName.
TypeName *string
// The details of the VPC of the Amazon ES destination.
VpcConfigurationDescription *VpcConfigurationDescription
noSmithyDocumentSerde
}
// Describes an update for a destination in Amazon OpenSearch Service.
type AmazonopensearchserviceDestinationUpdate struct {
// The buffering options. If no value is specified, AmazonopensearchBufferingHints
// object default values are used.
BufferingHints *AmazonopensearchserviceBufferingHints
// Describes the Amazon CloudWatch logging options for your delivery stream.
CloudWatchLoggingOptions *CloudWatchLoggingOptions
// The endpoint to use when communicating with the cluster. Specify either this
// ClusterEndpoint or the DomainARN field.
ClusterEndpoint *string
// Indicates the method for setting up document ID. The supported methods are
// Firehose generated document ID and OpenSearch Service generated document ID.
DocumentIdOptions *DocumentIdOptions
// The ARN of the Amazon OpenSearch Service domain. The IAM role must have
// permissions for DescribeDomain, DescribeDomains, and DescribeDomainConfig after
// assuming the IAM role specified in RoleARN.
DomainARN *string
// The Amazon OpenSearch Service index name.
IndexName *string
// The Amazon OpenSearch Service index rotation period. Index rotation appends a
// timestamp to IndexName to facilitate the expiration of old data.
IndexRotationPeriod AmazonopensearchserviceIndexRotationPeriod
// Describes a data processing configuration.
ProcessingConfiguration *ProcessingConfiguration
// The retry behavior in case Firehose is unable to deliver documents to Amazon
// OpenSearch Service. The default value is 300 (5 minutes).
RetryOptions *AmazonopensearchserviceRetryOptions
// The Amazon Resource Name (ARN) of the IAM role to be assumed by Firehose for
// calling the Amazon OpenSearch Service Configuration API and for indexing
// documents.
RoleARN *string
// Describes an update for a destination in Amazon S3.
S3Update *S3DestinationUpdate
// The Amazon OpenSearch Service type name. For Elasticsearch 6.x, there can be
// only one type per index. If you try to specify a new type for an existing index
// that already has another type, Firehose returns an error during runtime. If you
// upgrade Elasticsearch from 6.x to 7.x and don’t update your delivery stream,
// Firehose still delivers data to Elasticsearch with the old index name and type
// name. If you want to update your delivery stream with a new index name, provide
// an empty string for TypeName.
TypeName *string
noSmithyDocumentSerde
}
// Configures retry behavior in case Firehose is unable to deliver documents to
// Amazon OpenSearch Service.
type AmazonopensearchserviceRetryOptions struct {
// After an initial failure to deliver to Amazon OpenSearch Service, the total
// amount of time during which Firehose retries delivery (including the first
// attempt). After this time has elapsed, the failed documents are written to
// Amazon S3. Default value is 300 seconds (5 minutes). A value of 0 (zero) results
// in no retries.
DurationInSeconds *int32
noSmithyDocumentSerde
}
// The authentication configuration of the Amazon MSK cluster.
type AuthenticationConfiguration struct {
// The type of connectivity used to access the Amazon MSK cluster.
//
// This member is required.
Connectivity Connectivity
// The ARN of the role used to access the Amazon MSK cluster.
//
// This member is required.
RoleARN *string
noSmithyDocumentSerde
}
// Describes hints for the buffering to perform before delivering data to the
// destination. These options are treated as hints, and therefore Firehose might
// choose to use different values when it is optimal. The SizeInMBs and
// IntervalInSeconds parameters are optional. However, if specify a value for one
// of them, you must also provide a value for the other.
type BufferingHints struct {
// Buffer incoming data for the specified period of time, in seconds, before
// delivering it to the destination. The default value is 300. This parameter is
// optional but if you specify a value for it, you must also specify a value for
// SizeInMBs , and vice versa.
IntervalInSeconds *int32
// Buffer incoming data to the specified size, in MiBs, before delivering it to
// the destination. The default value is 5. This parameter is optional but if you
// specify a value for it, you must also specify a value for IntervalInSeconds ,
// and vice versa. We recommend setting this parameter to a value greater than the
// amount of data you typically ingest into the delivery stream in 10 seconds. For
// example, if you typically ingest data at 1 MiB/sec, the value should be 10 MiB
// or higher.
SizeInMBs *int32
noSmithyDocumentSerde
}
// Describes the Amazon CloudWatch logging options for your delivery stream.
type CloudWatchLoggingOptions struct {
// Enables or disables CloudWatch logging.
Enabled *bool
// The CloudWatch group name for logging. This value is required if CloudWatch
// logging is enabled.
LogGroupName *string
// The CloudWatch log stream name for logging. This value is required if
// CloudWatch logging is enabled.
LogStreamName *string
noSmithyDocumentSerde
}
// Describes a COPY command for Amazon Redshift.
type CopyCommand struct {
// The name of the target table. The table must already exist in the database.
//
// This member is required.
DataTableName *string
// Optional parameters to use with the Amazon Redshift COPY command. For more
// information, see the "Optional Parameters" section of Amazon Redshift COPY
// command (https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html) . Some
// possible examples that would apply to Firehose are as follows: delimiter '\t'
// lzop; - fields are delimited with "\t" (TAB character) and compressed using
// lzop. delimiter '|' - fields are delimited with "|" (this is the default
// delimiter). delimiter '|' escape - the delimiter should be escaped. fixedwidth
// 'venueid:3,venuename:25,venuecity:12,venuestate:2,venueseats:6' - fields are
// fixed width in the source, with each width specified after every column in the
// table. JSON 's3://mybucket/jsonpaths.txt' - data is in JSON format, and the
// path specified is the format of the data. For more examples, see Amazon
// Redshift COPY command examples (https://docs.aws.amazon.com/redshift/latest/dg/r_COPY_command_examples.html)
// .
CopyOptions *string
// A comma-separated list of column names.
DataTableColumns *string
noSmithyDocumentSerde
}
// Specifies that you want Firehose to convert data from the JSON format to the
// Parquet or ORC format before writing it to Amazon S3. Firehose uses the
// serializer and deserializer that you specify, in addition to the column
// information from the Amazon Web Services Glue table, to deserialize your input
// data from JSON and then serialize it to the Parquet or ORC format. For more
// information, see Firehose Record Format Conversion (https://docs.aws.amazon.com/firehose/latest/dev/record-format-conversion.html)
// .
type DataFormatConversionConfiguration struct {
// Defaults to true . Set it to false if you want to disable format conversion
// while preserving the configuration details.
Enabled *bool
// Specifies the deserializer that you want Firehose to use to convert the format
// of your data from JSON. This parameter is required if Enabled is set to true.
InputFormatConfiguration *InputFormatConfiguration
// Specifies the serializer that you want Firehose to use to convert the format of
// your data to the Parquet or ORC format. This parameter is required if Enabled
// is set to true.
OutputFormatConfiguration *OutputFormatConfiguration
// Specifies the Amazon Web Services Glue Data Catalog table that contains the
// column information. This parameter is required if Enabled is set to true.
SchemaConfiguration *SchemaConfiguration
noSmithyDocumentSerde
}
// Contains information about a delivery stream.
type DeliveryStreamDescription struct {
// The Amazon Resource Name (ARN) of the delivery stream. For more information,
// see Amazon Resource Names (ARNs) and Amazon Web Services Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// .
//
// This member is required.
DeliveryStreamARN *string
// The name of the delivery stream.
//
// This member is required.
DeliveryStreamName *string
// The status of the delivery stream. If the status of a delivery stream is
// CREATING_FAILED , this status doesn't change, and you can't invoke
// CreateDeliveryStream again on it. However, you can invoke the
// DeleteDeliveryStream operation to delete it.
//
// This member is required.
DeliveryStreamStatus DeliveryStreamStatus
// The delivery stream type. This can be one of the following values:
// - DirectPut : Provider applications access the delivery stream directly.
// - KinesisStreamAsSource : The delivery stream uses a Kinesis data stream as a
// source.
//
// This member is required.
DeliveryStreamType DeliveryStreamType
// The destinations.
//
// This member is required.
Destinations []DestinationDescription
// Indicates whether there are more destinations available to list.
//
// This member is required.
HasMoreDestinations *bool
// Each time the destination is updated for a delivery stream, the version ID is
// changed, and the current version ID is required when updating the destination.
// This is so that the service knows it is applying the changes to the correct
// version of the delivery stream.
//
// This member is required.
VersionId *string
// The date and time that the delivery stream was created.
CreateTimestamp *time.Time
// Indicates the server-side encryption (SSE) status for the delivery stream.
DeliveryStreamEncryptionConfiguration *DeliveryStreamEncryptionConfiguration
// Provides details in case one of the following operations fails due to an error
// related to KMS: CreateDeliveryStream , DeleteDeliveryStream ,
// StartDeliveryStreamEncryption , StopDeliveryStreamEncryption .
FailureDescription *FailureDescription
// The date and time that the delivery stream was last updated.
LastUpdateTimestamp *time.Time
// If the DeliveryStreamType parameter is KinesisStreamAsSource , a
// SourceDescription object describing the source Kinesis data stream.
Source *SourceDescription
noSmithyDocumentSerde
}
// Contains information about the server-side encryption (SSE) status for the
// delivery stream, the type customer master key (CMK) in use, if any, and the ARN
// of the CMK. You can get DeliveryStreamEncryptionConfiguration by invoking the
// DescribeDeliveryStream operation.
type DeliveryStreamEncryptionConfiguration struct {
// Provides details in case one of the following operations fails due to an error
// related to KMS: CreateDeliveryStream , DeleteDeliveryStream ,
// StartDeliveryStreamEncryption , StopDeliveryStreamEncryption .
FailureDescription *FailureDescription
// If KeyType is CUSTOMER_MANAGED_CMK , this field contains the ARN of the customer
// managed CMK. If KeyType is Amazon Web Services_OWNED_CMK ,
// DeliveryStreamEncryptionConfiguration doesn't contain a value for KeyARN .
KeyARN *string
// Indicates the type of customer master key (CMK) that is used for encryption.
// The default setting is Amazon Web Services_OWNED_CMK . For more information
// about CMKs, see Customer Master Keys (CMKs) (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys)
// .
KeyType KeyType
// This is the server-side encryption (SSE) status for the delivery stream. For a
// full description of the different values of this status, see
// StartDeliveryStreamEncryption and StopDeliveryStreamEncryption . If this status
// is ENABLING_FAILED or DISABLING_FAILED , it is the status of the most recent
// attempt to enable or disable SSE, respectively.
Status DeliveryStreamEncryptionStatus
noSmithyDocumentSerde
}
// Specifies the type and Amazon Resource Name (ARN) of the CMK to use for
// Server-Side Encryption (SSE).
type DeliveryStreamEncryptionConfigurationInput struct {
// Indicates the type of customer master key (CMK) to use for encryption. The
// default setting is Amazon Web Services_OWNED_CMK . For more information about
// CMKs, see Customer Master Keys (CMKs) (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys)
// . When you invoke CreateDeliveryStream or StartDeliveryStreamEncryption with
// KeyType set to CUSTOMER_MANAGED_CMK, Firehose invokes the Amazon KMS operation
// CreateGrant (https://docs.aws.amazon.com/kms/latest/APIReference/API_CreateGrant.html)
// to create a grant that allows the Firehose service to use the customer managed
// CMK to perform encryption and decryption. Firehose manages that grant. When you
// invoke StartDeliveryStreamEncryption to change the CMK for a delivery stream
// that is encrypted with a customer managed CMK, Firehose schedules the grant it
// had on the old CMK for retirement. You can use a CMK of type
// CUSTOMER_MANAGED_CMK to encrypt up to 500 delivery streams. If a
// CreateDeliveryStream or StartDeliveryStreamEncryption operation exceeds this
// limit, Firehose throws a LimitExceededException . To encrypt your delivery
// stream, use symmetric CMKs. Firehose doesn't support asymmetric CMKs. For
// information about symmetric and asymmetric CMKs, see About Symmetric and
// Asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/symm-asymm-concepts.html)
// in the Amazon Web Services Key Management Service developer guide.
//
// This member is required.
KeyType KeyType
// If you set KeyType to CUSTOMER_MANAGED_CMK , you must specify the Amazon
// Resource Name (ARN) of the CMK. If you set KeyType to Amazon Web
// Services_OWNED_CMK , Firehose uses a service-account CMK.
KeyARN *string
noSmithyDocumentSerde
}
// The deserializer you want Firehose to use for converting the input data from
// JSON. Firehose then serializes the data to its final format using the Serializer
// . Firehose supports two types of deserializers: the Apache Hive JSON SerDe (https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-JSON)
// and the OpenX JSON SerDe (https://github.com/rcongiu/Hive-JSON-Serde) .
type Deserializer struct {
// The native Hive / HCatalog JsonSerDe. Used by Firehose for deserializing data,
// which means converting it from the JSON format in preparation for serializing it
// to the Parquet or ORC format. This is one of two deserializers you can choose,
// depending on which one offers the functionality you need. The other option is
// the OpenX SerDe.
HiveJsonSerDe *HiveJsonSerDe
// The OpenX SerDe. Used by Firehose for deserializing data, which means
// converting it from the JSON format in preparation for serializing it to the
// Parquet or ORC format. This is one of two deserializers you can choose,
// depending on which one offers the functionality you need. The other option is
// the native Hive / HCatalog JsonSerDe.
OpenXJsonSerDe *OpenXJsonSerDe
noSmithyDocumentSerde
}
// Describes the destination for a delivery stream.
type DestinationDescription struct {
// The ID of the destination.
//
// This member is required.
DestinationId *string
// The destination in the Serverless offering for Amazon OpenSearch Service.
AmazonOpenSearchServerlessDestinationDescription *AmazonOpenSearchServerlessDestinationDescription
// The destination in Amazon OpenSearch Service.
AmazonopensearchserviceDestinationDescription *AmazonopensearchserviceDestinationDescription
// The destination in Amazon ES.
ElasticsearchDestinationDescription *ElasticsearchDestinationDescription
// The destination in Amazon S3.
ExtendedS3DestinationDescription *ExtendedS3DestinationDescription
// Describes the specified HTTP endpoint destination.
HttpEndpointDestinationDescription *HttpEndpointDestinationDescription
// The destination in Amazon Redshift.
RedshiftDestinationDescription *RedshiftDestinationDescription
// [Deprecated] The destination in Amazon S3.
S3DestinationDescription *S3DestinationDescription
// Optional description for the destination
SnowflakeDestinationDescription *SnowflakeDestinationDescription
// The destination in Splunk.
SplunkDestinationDescription *SplunkDestinationDescription
noSmithyDocumentSerde
}
// Indicates the method for setting up document ID. The supported methods are
// Firehose generated document ID and OpenSearch Service generated document ID.
type DocumentIdOptions struct {
// When the FIREHOSE_DEFAULT option is chosen, Firehose generates a unique
// document ID for each record based on a unique internal identifier. The generated
// document ID is stable across multiple delivery attempts, which helps prevent the
// same record from being indexed multiple times with different document IDs. When
// the NO_DOCUMENT_ID option is chosen, Firehose does not include any document IDs
// in the requests it sends to the Amazon OpenSearch Service. This causes the
// Amazon OpenSearch Service domain to generate document IDs. In case of multiple
// delivery attempts, this may cause the same record to be indexed more than once
// with different document IDs. This option enables write-heavy operations, such as
// the ingestion of logs and observability data, to consume less resources in the
// Amazon OpenSearch Service domain, resulting in improved performance.
//
// This member is required.
DefaultDocumentIdFormat DefaultDocumentIdFormat
noSmithyDocumentSerde
}
// The configuration of the dynamic partitioning mechanism that creates smaller
// data sets from the streaming data by partitioning it based on partition keys.
// Currently, dynamic partitioning is only supported for Amazon S3 destinations.
type DynamicPartitioningConfiguration struct {
// Specifies that the dynamic partitioning is enabled for this Firehose delivery
// stream.
Enabled *bool
// The retry behavior in case Firehose is unable to deliver data to an Amazon S3
// prefix.
RetryOptions *RetryOptions
noSmithyDocumentSerde
}
// Describes the buffering to perform before delivering data to the Amazon ES
// destination.
type ElasticsearchBufferingHints struct {
// Buffer incoming data for the specified period of time, in seconds, before
// delivering it to the destination. The default value is 300 (5 minutes).
IntervalInSeconds *int32
// Buffer incoming data to the specified size, in MBs, before delivering it to the
// destination. The default value is 5. We recommend setting this parameter to a
// value greater than the amount of data you typically ingest into the delivery
// stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec, the
// value should be 10 MB or higher.
SizeInMBs *int32
noSmithyDocumentSerde
}
// Describes the configuration of a destination in Amazon ES.
type ElasticsearchDestinationConfiguration struct {
// The Elasticsearch index name.
//
// This member is required.
IndexName *string
// The Amazon Resource Name (ARN) of the IAM role to be assumed by Firehose for
// calling the Amazon ES Configuration API and for indexing documents. For more
// information, see Grant Firehose Access to an Amazon S3 Destination (https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html#using-iam-s3)
// and Amazon Resource Names (ARNs) and Amazon Web Services Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// .
//
// This member is required.
RoleARN *string
// The configuration for the backup Amazon S3 location.
//
// This member is required.
S3Configuration *S3DestinationConfiguration
// The buffering options. If no value is specified, the default values for
// ElasticsearchBufferingHints are used.
BufferingHints *ElasticsearchBufferingHints
// The Amazon CloudWatch logging options for your delivery stream.
CloudWatchLoggingOptions *CloudWatchLoggingOptions
// The endpoint to use when communicating with the cluster. Specify either this
// ClusterEndpoint or the DomainARN field.
ClusterEndpoint *string
// Indicates the method for setting up document ID. The supported methods are
// Firehose generated document ID and OpenSearch Service generated document ID.
DocumentIdOptions *DocumentIdOptions
// The ARN of the Amazon ES domain. The IAM role must have permissions for
// DescribeDomain , DescribeDomains , and DescribeDomainConfig after assuming the
// role specified in RoleARN. For more information, see Amazon Resource Names
// (ARNs) and Amazon Web Services Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// . Specify either ClusterEndpoint or DomainARN .
DomainARN *string
// The Elasticsearch index rotation period. Index rotation appends a timestamp to
// the IndexName to facilitate the expiration of old data. For more information,
// see Index Rotation for the Amazon ES Destination (https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#es-index-rotation)
// . The default value is OneDay .
IndexRotationPeriod ElasticsearchIndexRotationPeriod
// The data processing configuration.
ProcessingConfiguration *ProcessingConfiguration
// The retry behavior in case Firehose is unable to deliver documents to Amazon
// ES. The default value is 300 (5 minutes).
RetryOptions *ElasticsearchRetryOptions
// Defines how documents should be delivered to Amazon S3. When it is set to
// FailedDocumentsOnly , Firehose writes any documents that could not be indexed to
// the configured Amazon S3 destination, with AmazonOpenSearchService-failed/
// appended to the key prefix. When set to AllDocuments , Firehose delivers all
// incoming records to Amazon S3, and also writes failed documents with
// AmazonOpenSearchService-failed/ appended to the prefix. For more information,
// see Amazon S3 Backup for the Amazon ES Destination (https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#es-s3-backup)
// . Default value is FailedDocumentsOnly . You can't change this backup mode after
// you create the delivery stream.
S3BackupMode ElasticsearchS3BackupMode
// The Elasticsearch type name. For Elasticsearch 6.x, there can be only one type
// per index. If you try to specify a new type for an existing index that already
// has another type, Firehose returns an error during run time. For Elasticsearch
// 7.x, don't specify a TypeName .
TypeName *string
// The details of the VPC of the Amazon destination.
VpcConfiguration *VpcConfiguration
noSmithyDocumentSerde
}
// The destination description in Amazon ES.
type ElasticsearchDestinationDescription struct {
// The buffering options.
BufferingHints *ElasticsearchBufferingHints
// The Amazon CloudWatch logging options.
CloudWatchLoggingOptions *CloudWatchLoggingOptions
// The endpoint to use when communicating with the cluster. Firehose uses either
// this ClusterEndpoint or the DomainARN field to send data to Amazon ES.
ClusterEndpoint *string
// Indicates the method for setting up document ID. The supported methods are
// Firehose generated document ID and OpenSearch Service generated document ID.
DocumentIdOptions *DocumentIdOptions
// The ARN of the Amazon ES domain. For more information, see Amazon Resource
// Names (ARNs) and Amazon Web Services Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// . Firehose uses either ClusterEndpoint or DomainARN to send data to Amazon ES.
DomainARN *string
// The Elasticsearch index name.
IndexName *string
// The Elasticsearch index rotation period
IndexRotationPeriod ElasticsearchIndexRotationPeriod
// The data processing configuration.
ProcessingConfiguration *ProcessingConfiguration
// The Amazon ES retry options.
RetryOptions *ElasticsearchRetryOptions
// The Amazon Resource Name (ARN) of the Amazon Web Services credentials. For more
// information, see Amazon Resource Names (ARNs) and Amazon Web Services Service
// Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// .
RoleARN *string
// The Amazon S3 backup mode.
S3BackupMode ElasticsearchS3BackupMode
// The Amazon S3 destination.
S3DestinationDescription *S3DestinationDescription
// The Elasticsearch type name. This applies to Elasticsearch 6.x and lower
// versions. For Elasticsearch 7.x and OpenSearch Service 1.x, there's no value for
// TypeName .
TypeName *string
// The details of the VPC of the Amazon OpenSearch or the Amazon OpenSearch
// Serverless destination.
VpcConfigurationDescription *VpcConfigurationDescription
noSmithyDocumentSerde
}
// Describes an update for a destination in Amazon ES.
type ElasticsearchDestinationUpdate struct {
// The buffering options. If no value is specified, ElasticsearchBufferingHints
// object default values are used.
BufferingHints *ElasticsearchBufferingHints
// The CloudWatch logging options for your delivery stream.
CloudWatchLoggingOptions *CloudWatchLoggingOptions
// The endpoint to use when communicating with the cluster. Specify either this
// ClusterEndpoint or the DomainARN field.
ClusterEndpoint *string
// Indicates the method for setting up document ID. The supported methods are
// Firehose generated document ID and OpenSearch Service generated document ID.
DocumentIdOptions *DocumentIdOptions
// The ARN of the Amazon ES domain. The IAM role must have permissions for
// DescribeDomain , DescribeDomains , and DescribeDomainConfig after assuming the
// IAM role specified in RoleARN . For more information, see Amazon Resource Names
// (ARNs) and Amazon Web Services Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// . Specify either ClusterEndpoint or DomainARN .
DomainARN *string
// The Elasticsearch index name.
IndexName *string
// The Elasticsearch index rotation period. Index rotation appends a timestamp to
// IndexName to facilitate the expiration of old data. For more information, see
// Index Rotation for the Amazon ES Destination (https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#es-index-rotation)
// . Default value is OneDay .
IndexRotationPeriod ElasticsearchIndexRotationPeriod
// The data processing configuration.
ProcessingConfiguration *ProcessingConfiguration
// The retry behavior in case Firehose is unable to deliver documents to Amazon
// ES. The default value is 300 (5 minutes).
RetryOptions *ElasticsearchRetryOptions
// The Amazon Resource Name (ARN) of the IAM role to be assumed by Firehose for
// calling the Amazon ES Configuration API and for indexing documents. For more
// information, see Grant Firehose Access to an Amazon S3 Destination (https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html#using-iam-s3)
// and Amazon Resource Names (ARNs) and Amazon Web Services Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// .
RoleARN *string
// The Amazon S3 destination.
S3Update *S3DestinationUpdate
// The Elasticsearch type name. For Elasticsearch 6.x, there can be only one type
// per index. If you try to specify a new type for an existing index that already
// has another type, Firehose returns an error during runtime. If you upgrade
// Elasticsearch from 6.x to 7.x and don’t update your delivery stream, Firehose
// still delivers data to Elasticsearch with the old index name and type name. If
// you want to update your delivery stream with a new index name, provide an empty
// string for TypeName .
TypeName *string
noSmithyDocumentSerde
}
// Configures retry behavior in case Firehose is unable to deliver documents to
// Amazon ES.
type ElasticsearchRetryOptions struct {
// After an initial failure to deliver to Amazon ES, the total amount of time
// during which Firehose retries delivery (including the first attempt). After this
// time has elapsed, the failed documents are written to Amazon S3. Default value
// is 300 seconds (5 minutes). A value of 0 (zero) results in no retries.
DurationInSeconds *int32
noSmithyDocumentSerde
}
// Describes the encryption for a destination in Amazon S3.
type EncryptionConfiguration struct {
// The encryption key.
KMSEncryptionConfig *KMSEncryptionConfig
// Specifically override existing encryption information to ensure that no
// encryption is used.
NoEncryptionConfig NoEncryptionConfig
noSmithyDocumentSerde
}
// Describes the configuration of a destination in Amazon S3.
type ExtendedS3DestinationConfiguration struct {
// The ARN of the S3 bucket. For more information, see Amazon Resource Names
// (ARNs) and Amazon Web Services Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// .
//
// This member is required.
BucketARN *string
// The Amazon Resource Name (ARN) of the Amazon Web Services credentials. For more
// information, see Amazon Resource Names (ARNs) and Amazon Web Services Service
// Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// .
//
// This member is required.
RoleARN *string
// The buffering option.
BufferingHints *BufferingHints