Publish new connectors to log offending json fields in json to avro c…

…onversion for debugging (#12578) * Publish new connectors to log debugging info in json to avro conversion * Add pull request id * auto-bump connector version * auto-bump connector version * auto-bump connector version * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · May 4, 2022 · bd59726 · bd59726
1 parent 22b67d8
commit bd59726
Show file tree

Hide file tree

Showing 11 changed files with 89 additions and 72 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -24,7 +24,7 @@
 - name: BigQuery
   destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
   dockerRepository: airbyte/destination-bigquery
-  dockerImageTag: 1.1.3
+  dockerImageTag: 1.1.4
   documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
   icon: bigquery.svg
   resourceRequirements:
@@ -36,7 +36,7 @@
 - name: BigQuery (denormalized typed struct)
   destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496
   dockerRepository: airbyte/destination-bigquery-denormalized
-  dockerImageTag: 0.3.3
+  dockerImageTag: 0.3.4
   documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
   icon: bigquery.svg
   resourceRequirements:
@@ -83,7 +83,7 @@
 - name: Google Cloud Storage (GCS)
   destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a
   dockerRepository: airbyte/destination-gcs
-  dockerImageTag: 0.2.4
+  dockerImageTag: 0.2.5
   documentationUrl: https://docs.airbyte.io/integrations/destinations/gcs
   icon: googlecloudstorage.svg
   resourceRequirements:
@@ -208,7 +208,7 @@
 - name: S3
   destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
   dockerRepository: airbyte/destination-s3
-  dockerImageTag: 0.3.3
+  dockerImageTag: 0.3.4
   documentationUrl: https://docs.airbyte.io/integrations/destinations/s3
   icon: s3.svg
   resourceRequirements:

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -285,7 +285,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-bigquery:1.1.3"
+- dockerImage: "airbyte/destination-bigquery:1.1.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery"
     connectionSpecification:
@@ -494,7 +494,7 @@
     - "overwrite"
     - "append"
     - "append_dedup"
-- dockerImage: "airbyte/destination-bigquery-denormalized:0.3.3"
+- dockerImage: "airbyte/destination-bigquery-denormalized:0.3.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery"
     connectionSpecification:
@@ -1287,7 +1287,7 @@
     - "overwrite"
     - "append"
     supportsNamespaces: true
-- dockerImage: "airbyte/destination-gcs:0.2.4"
+- dockerImage: "airbyte/destination-gcs:0.2.5"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/gcs"
     connectionSpecification:
@@ -3643,7 +3643,7 @@
     supported_destination_sync_modes:
     - "append"
     - "overwrite"
-- dockerImage: "airbyte/destination-s3:0.3.3"
+- dockerImage: "airbyte/destination-s3:0.3.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/s3"
     connectionSpecification:
@@ -3657,40 +3657,46 @@
       - "format"
       additionalProperties: false
       properties:
-        s3_endpoint:
-          title: "Endpoint"
+        access_key_id:
           type: "string"
-          default: ""
-          description: "This is your S3 endpoint url.(if you are working with AWS\
-            \ S3, just leave empty)."
+          description: "The access key ID to access the S3 bucket. Airbyte requires\
+            \ Read and Write permissions to the given bucket. See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys)\
+            \ on how to generate an access key."
+          title: "S3 Key ID *"
+          airbyte_secret: true
           examples:
-          - "http://localhost:9000"
+          - "A012345678910EXAMPLE"
           order: 0
-        s3_bucket_name:
-          title: "S3 Bucket Name"
+        secret_access_key:
           type: "string"
-          description: "The name of the S3 bucket."
+          description: "The corresponding secret to the access key ID. See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys)"
+          title: "S3 Access Key *"
+          airbyte_secret: true
           examples:
-          - "airbyte_sync"
+          - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
           order: 1
-        s3_bucket_path:
-          description: "Directory under the S3 bucket where data will be written."
+        s3_bucket_name:
+          title: "S3 Bucket Name *"
           type: "string"
+          description: "The name of the S3 bucket. See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html)\
+            \ to create an S3 bucket."
           examples:
-          - "data_sync/test"
+          - "airbyte_sync"
           order: 2
-        s3_path_format:
-          description: "Format string on how data will be organized inside the S3\
-            \ bucket directory"
+        s3_bucket_path:
+          title: "S3 Bucket Path *"
+          description: "Directory under the S3 bucket where data will be written.\
+            \ See [this](https://docs.airbyte.com/integrations/destinations/s3#:~:text=to%20format%20the-,bucket%20path,-%3A)"
           type: "string"
           examples:
-          - "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
+          - "data_sync/test"
           order: 3
         s3_bucket_region:
-          title: "S3 Bucket Region"
+          title: "S3 Bucket Region *"
           type: "string"
           default: ""
-          description: "The region of the S3 bucket."
+          description: "The region of the S3 bucket. See [this](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions)\
+            \ for all region codes."
           enum:
           - ""
           - "us-east-1"
@@ -3719,47 +3725,31 @@
           - "us-gov-east-1"
           - "us-gov-west-1"
           order: 4
-        access_key_id:
-          type: "string"
-          description: "The access key id to access the S3 bucket. Airbyte requires\
-            \ Read and Write permissions to the given bucket, if not set, Airbyte\
-            \ will rely on Instance Profile."
-          title: "S3 Key Id"
-          airbyte_secret: true
-          examples:
-          - "A012345678910EXAMPLE"
-          order: 5
-        secret_access_key:
-          type: "string"
-          description: "The corresponding secret to the access key id, if S3 Key Id\
-            \ is set, then S3 Access Key must also be provided"
-          title: "S3 Access Key"
-          airbyte_secret: true
-          examples:
-          - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
-          order: 6
         format:
-          title: "Output Format"
+          title: "Output Format *"
           type: "object"
-          description: "Output data format"
+          description: "Format of the data output. See [this](https://docs.airbyte.com/integrations/destinations/s3/#output-schema)\
+            \ for more details"
           oneOf:
           - title: "Avro: Apache Avro"
             required:
             - "format_type"
             - "compression_codec"
             properties:
               format_type:
+                title: "Format Type *"
                 type: "string"
                 enum:
                 - "Avro"
                 default: "Avro"
+                order: 0
               compression_codec:
-                title: "Compression Codec"
+                title: "Compression Codec *"
                 description: "The compression algorithm used to compress data. Default\
                   \ to no compression."
                 type: "object"
                 oneOf:
-                - title: "no compression"
+                - title: "No Compression"
                   required:
                   - "codec"
                   properties:
@@ -3779,7 +3769,7 @@
                       - "Deflate"
                       default: "Deflate"
                     compression_level:
-                      title: "Deflate level"
+                      title: "Deflate Level"
                       description: "0: no compression & fastest, 9: best compression\
                         \ & slowest."
                       type: "integer"
@@ -3806,7 +3796,7 @@
                       - "xz"
                       default: "xz"
                     compression_level:
-                      title: "Compression level"
+                      title: "Compression Level"
                       description: "See <a href=\"https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/compressors/xz/XZCompressorOutputStream.html#XZCompressorOutputStream-java.io.OutputStream-int-\"\
                         >here</a> for details."
                       type: "integer"
@@ -3824,7 +3814,7 @@
                       - "zstandard"
                       default: "zstandard"
                     compression_level:
-                      title: "Compression level"
+                      title: "Compression Level"
                       description: "Negative levels are 'fast' modes akin to lz4 or\
                         \ snappy, levels above 9 are generally for archival purposes,\
                         \ and levels above 18 use a lot of memory."
@@ -3833,7 +3823,7 @@
                       minimum: -5
                       maximum: 22
                     include_checksum:
-                      title: "Include checksum"
+                      title: "Include Checksum"
                       description: "If true, include a checksum with each data block."
                       type: "boolean"
                       default: false
@@ -3846,8 +3836,9 @@
                       enum:
                       - "snappy"
                       default: "snappy"
+                order: 1
               part_size_mb:
-                title: "Block Size (MB) for Amazon S3 multipart upload"
+                title: "Block Size (MB) for Amazon S3 multipart upload (Optional)"
                 description: "This is the size of a \"Part\" being buffered in memory.\
                   \ It limits the memory usage when writing. Larger values will allow\
                   \ to upload a bigger files and improve the speed, but consumes9\
@@ -3856,12 +3847,14 @@
                 default: 5
                 examples:
                 - 5
+                order: 2
           - title: "CSV: Comma-Separated Values"
             required:
             - "format_type"
             - "flattening"
             properties:
               format_type:
+                title: "Format Type *"
                 type: "string"
                 enum:
                 - "CSV"
@@ -3876,7 +3869,7 @@
                 - "No flattening"
                 - "Root level flattening"
               part_size_mb:
-                title: "Block Size (MB) for Amazon S3 multipart upload"
+                title: "Block Size (MB) For Amazon S3 Multipart Upload (Optional)"
                 description: "This is the size of a \"Part\" being buffered in memory.\
                   \ It limits the memory usage when writing. Larger values will allow\
                   \ to upload a bigger files and improve the speed, but consumes9\
@@ -3910,17 +3903,18 @@
                       enum:
                       - "GZIP"
                       default: "GZIP"
-          - title: "JSON Lines: newline-delimited JSON"
+          - title: "JSON Lines: Newline-delimited JSON"
             required:
             - "format_type"
             properties:
               format_type:
+                title: "Format Type *"
                 type: "string"
                 enum:
                 - "JSONL"
                 default: "JSONL"
               part_size_mb:
-                title: "Block Size (MB) for Amazon S3 multipart upload"
+                title: "Block Size (MB) For Amazon S3 Multipart Upload (Optional)"
                 description: "This is the size of a \"Part\" being buffered in memory.\
                   \ It limits the memory usage when writing. Larger values will allow\
                   \ to upload a bigger files and improve the speed, but consumes9\
@@ -3957,12 +3951,13 @@
             - "format_type"
             properties:
               format_type:
+                title: "Format Type *"
                 type: "string"
                 enum:
                 - "Parquet"
                 default: "Parquet"
               compression_codec:
-                title: "Compression Codec"
+                title: "Compression Codec (Optional)"
                 description: "The compression algorithm used to compress data pages."
                 type: "string"
                 enum:
@@ -3975,7 +3970,7 @@
                 - "ZSTD"
                 default: "UNCOMPRESSED"
               block_size_mb:
-                title: "Block Size (Row Group Size) (MB)"
+                title: "Block Size (Row Group Size) (MB) (Optional)"
                 description: "This is the size of a row group being buffered in memory.\
                   \ It limits the memory usage when writing. Larger values will improve\
                   \ the IO when reading, but consume more memory when writing. Default:\
@@ -3985,15 +3980,15 @@
                 examples:
                 - 128
               max_padding_size_mb:
-                title: "Max Padding Size (MB)"
+                title: "Max Padding Size (MB) (Optional)"
                 description: "Maximum size allowed as padding to align row groups.\
                   \ This is also the minimum size of a row group. Default: 8 MB."
                 type: "integer"
                 default: 8
                 examples:
                 - 8
               page_size_kb:
-                title: "Page Size (KB)"
+                title: "Page Size (KB) (Optional)"
                 description: "The page size is for compression. A block is composed\
                   \ of pages. A page is the smallest unit that must be read fully\
                   \ to access a single record. If this value is too small, the compression\
@@ -4003,7 +3998,7 @@
                 examples:
                 - 1024
               dictionary_page_size_kb:
-                title: "Dictionary Page Size (KB)"
+                title: "Dictionary Page Size (KB) (Optional)"
                 description: "There is one dictionary page per column per row group\
                   \ when dictionary encoding is used. The dictionary page size works\
                   \ like the page size but for dictionary. Default: 1024 KB."
@@ -4012,10 +4007,27 @@
                 examples:
                 - 1024
               dictionary_encoding:
-                title: "Dictionary Encoding"
+                title: "Dictionary Encoding (Optional)"
                 description: "Default: true."
                 type: "boolean"
                 default: true
+          order: 5
+        s3_endpoint:
+          title: "Endpoint (Optional)"
+          type: "string"
+          default: ""
+          description: "This is your S3 endpoint url. (If you are working with AWS\
+            \ S3, you can leave blank). See [this](https://docs.aws.amazon.com/general/latest/gr/s3.html#:~:text=Service%20endpoints-,Amazon%20S3%20endpoints,-When%20you%20use)"
+          examples:
+          - "http://localhost:9000"
+          order: 6
+        s3_path_format:
+          title: "S3 Path Format (Optional)"
+          description: "Format string on how data will be organized inside the S3\
+            \ bucket directory. See [this](https://docs.airbyte.com/integrations/destinations/s3#:~:text=The%20full%20path%20of%20the%20output%20data%20with%20the%20default%20S3%20path%20format)"
+          type: "string"
+          examples:
+          - "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
           order: 7
     supportsIncremental: true
     supportsNormalization: false

diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile
@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.3.3
+LABEL io.airbyte.version=0.3.4
 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized
diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile
@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=1.1.3
+LABEL io.airbyte.version=1.1.4
 LABEL io.airbyte.name=airbyte/destination-bigquery
diff --git a/airbyte-integrations/connectors/destination-databricks/Dockerfile b/airbyte-integrations/connectors/destination-databricks/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-databricks
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.4
+LABEL io.airbyte.version=0.1.5
 LABEL io.airbyte.name=airbyte/destination-databricks
diff --git a/airbyte-integrations/connectors/destination-gcs/Dockerfile b/airbyte-integrations/connectors/destination-gcs/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.2.4
+LABEL io.airbyte.version=0.2.5
 LABEL io.airbyte.name=airbyte/destination-gcs
diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-s3
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.3.3
+LABEL io.airbyte.version=0.3.4
 LABEL io.airbyte.name=airbyte/destination-s3