Deprecate s3 connection bucket arn (#873)

nucleuscloud · Dec 15, 2023 · b98da9f · b98da9f
1 parent e2e127f
commit b98da9f
Show file tree

Hide file tree

Showing 12 changed files with 323 additions and 279 deletions.
diff --git a/backend/gen/go/protos/mgmt/v1alpha1/connection.pb.go b/backend/gen/go/protos/mgmt/v1alpha1/connection.pb.go
diff --git a/backend/gen/go/protos/mgmt/v1alpha1/connection.pb.validate.go b/backend/gen/go/protos/mgmt/v1alpha1/connection.pb.validate.go
diff --git a/backend/protos/mgmt/v1alpha1/connection.proto b/backend/protos/mgmt/v1alpha1/connection.proto
@@ -119,11 +119,12 @@ message MysqlConnectionConfig {
 }
 
 message AwsS3ConnectionConfig {
-  string bucket_arn = 1;
+  string bucket_arn = 1 [deprecated = true];
   optional string path_prefix = 2;
   optional AwsS3Credentials credentials = 3;
   optional string region = 4;
   optional string endpoint = 5;
+  string bucket = 6 [(buf.validate.field).string.min_len = 1];
 }
 
 // S3 Credentials that are used by the worker process.

diff --git a/backend/sql/postgresql/models/models.go b/backend/sql/postgresql/models/models.go
@@ -2,6 +2,7 @@ package pg_models
 
 import (
 	"fmt"
+	"strings"
 
 	mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1"
 )
@@ -193,23 +194,29 @@ func (a *AwsS3Credentials) FromDto(dto *mgmtv1alpha1.AwsS3Credentials) {
 
 type AwsS3ConnectionConfig struct {
 	BucketArn   string
+	Bucket      string
 	PathPrefix  *string
 	Credentials *AwsS3Credentials
 	Region      *string
 	Endpoint    *string
 }
 
 func (a *AwsS3ConnectionConfig) ToDto() *mgmtv1alpha1.AwsS3ConnectionConfig {
+	var bucket = a.Bucket
+	if a.Bucket == "" && a.BucketArn != "" {
+		bucket = strings.ReplaceAll(a.BucketArn, "arn:aws:s3:::", "")
+	}
+
 	return &mgmtv1alpha1.AwsS3ConnectionConfig{
-		BucketArn:   a.BucketArn,
+		Bucket:      bucket,
 		PathPrefix:  a.PathPrefix,
 		Credentials: a.Credentials.ToDto(),
 		Region:      a.Region,
 		Endpoint:    a.Endpoint,
 	}
 }
 func (a *AwsS3ConnectionConfig) FromDto(dto *mgmtv1alpha1.AwsS3ConnectionConfig) error {
-	a.BucketArn = dto.BucketArn
+	a.Bucket = dto.Bucket
 	a.PathPrefix = dto.PathPrefix
 	a.Credentials = &AwsS3Credentials{}
 	a.Credentials.FromDto(dto.Credentials)

diff --git a/docs/docs/connections/s3.mdx b/docs/docs/connections/s3.mdx
@@ -22,7 +22,7 @@ This document outlines the steps to configure an AWS S3 data connection, which i
 
 **Connection Name**: Enter a unique name for this connection that you'll easily recognize. This is just a label and does not affect the connection itself.
 
-**Bucket ARN**: Input the Amazon Resource Name (ARN) of your S3 bucket.
+**Bucket**: Input the name of your S3 bucket.
 
 **Path Prefix**: Specify the path prefix of the bucket if necessary.
 

diff --git a/docs/protos/data/proto_docs.json b/docs/protos/data/proto_docs.json
@@ -878,7 +878,10 @@
               "ismap": false,
               "isoneof": false,
               "oneofdecl": "",
-              "defaultValue": ""
+              "defaultValue": "",
+              "options": {
+                "deprecated": true
+              }
             },
             {
               "name": "path_prefix",
@@ -927,6 +930,18 @@
               "isoneof": true,
               "oneofdecl": "_endpoint",
               "defaultValue": ""
+            },
+            {
+              "name": "bucket",
+              "description": "",
+              "label": "",
+              "type": "string",
+              "longType": "string",
+              "fullType": "string",
+              "ismap": false,
+              "isoneof": false,
+              "oneofdecl": "",
+              "defaultValue": ""
             }
           ]
         },

diff --git a/frontend/apps/web/app/[account]/connections/[id]/components/AwsS3Form.tsx b/frontend/apps/web/app/[account]/connections/[id]/components/AwsS3Form.tsx
@@ -111,16 +111,16 @@ export default function AwsS3Form(props: Props) {
         />
         <FormField
           control={form.control}
-          name="s3.bucketArn"
+          name="s3.bucket"
           render={({ field }) => (
             <FormItem>
               <FormLabel>
                 <RequiredLabel />
-                Bucket ARN
+                Bucket
               </FormLabel>
-              <FormDescription>The bucket ARN</FormDescription>
+              <FormDescription>The bucket</FormDescription>
               <FormControl>
-                <Input placeholder="Bucket ARN" {...field} />
+                <Input placeholder="Bucket" {...field} />
               </FormControl>
               <FormMessage />
             </FormItem>
@@ -314,7 +314,7 @@ async function updateAwsS3Connection(
             config: {
               case: 'awsS3Config',
               value: new AwsS3ConnectionConfig({
-                bucketArn: s3.bucketArn,
+                bucket: s3.bucket,
                 pathPrefix: s3.pathPrefix,
                 region: s3.region,
                 endpoint: s3.endpoint,

diff --git a/frontend/apps/web/app/[account]/connections/[id]/components/connection-component.tsx b/frontend/apps/web/app/[account]/connections/[id]/components/connection-component.tsx
@@ -155,7 +155,7 @@ export function getConnectionComponentDetails(
             defaultValues={{
               connectionName: connection.name,
               s3: {
-                bucketArn: connection.connectionConfig.config.value.bucketArn,
+                bucket: connection.connectionConfig.config.value.bucket,
                 pathPrefix: connection.connectionConfig.config.value.pathPrefix,
                 credentials: {
                   accessKeyId:

diff --git a/frontend/apps/web/app/[account]/new/connection/aws-s3/AwsS3Form.tsx b/frontend/apps/web/app/[account]/new/connection/aws-s3/AwsS3Form.tsx
@@ -37,7 +37,7 @@ export default function AwsS3Form() {
     defaultValues: {
       connectionName: '',
       s3: {
-        bucketArn: '',
+        bucket: '',
       },
     },
     context: { accountId: account?.id ?? '' },
@@ -114,16 +114,16 @@ export default function AwsS3Form() {
         />
         <FormField
           control={form.control}
-          name="s3.bucketArn"
+          name="s3.bucket"
           render={({ field }) => (
             <FormItem>
               <FormLabel>
                 <RequiredLabel />
-                Bucket ARN
+                Bucket
               </FormLabel>
-              <FormDescription>The bucket ARN</FormDescription>
+              <FormDescription>The bucket</FormDescription>
               <FormControl>
-                <Input placeholder="Bucket ARN" {...field} />
+                <Input placeholder="Bucket" {...field} />
               </FormControl>
               <FormMessage />
             </FormItem>
@@ -314,7 +314,7 @@ async function createAwsS3Connection(
           config: {
             case: 'awsS3Config',
             value: new AwsS3ConnectionConfig({
-              bucketArn: s3.bucketArn,
+              bucket: s3.bucket,
               pathPrefix: s3.pathPrefix,
               region: s3.region,
               endpoint: s3.endpoint,

diff --git a/frontend/apps/web/yup-validations/connections.ts b/frontend/apps/web/yup-validations/connections.ts
@@ -108,7 +108,7 @@ export type PostgresFormValues = Yup.InferType<typeof POSTGRES_FORM_SCHEMA>;
 export const AWS_FORM_SCHEMA = Yup.object({
   connectionName: connectionNameSchema,
   s3: Yup.object({
-    bucketArn: Yup.string().required(),
+    bucket: Yup.string().required(),
     pathPrefix: Yup.string().optional(),
     region: Yup.string().optional(),
     endpoint: Yup.string().optional(),

diff --git a/frontend/packages/sdk/src/client/mgmt/v1alpha1/connection_pb.ts b/frontend/packages/sdk/src/client/mgmt/v1alpha1/connection_pb.ts
@@ -859,7 +859,8 @@ export class MysqlConnectionConfig extends Message<MysqlConnectionConfig> {
  */
 export class AwsS3ConnectionConfig extends Message<AwsS3ConnectionConfig> {
   /**
-   * @generated from field: string bucket_arn = 1;
+   * @generated from field: string bucket_arn = 1 [deprecated = true];
+   * @deprecated
    */
   bucketArn = "";
 
@@ -883,6 +884,11 @@ export class AwsS3ConnectionConfig extends Message<AwsS3ConnectionConfig> {
    */
   endpoint?: string;
 
+  /**
+   * @generated from field: string bucket = 6;
+   */
+  bucket = "";
+
   constructor(data?: PartialMessage<AwsS3ConnectionConfig>) {
     super();
     proto3.util.initPartial(data, this);
@@ -896,6 +902,7 @@ export class AwsS3ConnectionConfig extends Message<AwsS3ConnectionConfig> {
     { no: 3, name: "credentials", kind: "message", T: AwsS3Credentials, opt: true },
     { no: 4, name: "region", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
     { no: 5, name: "endpoint", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
+    { no: 6, name: "bucket", kind: "scalar", T: 9 /* ScalarType.STRING */ },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): AwsS3ConnectionConfig {

diff --git a/worker/pkg/workflows/datasync/activities/benthos-builder.go b/worker/pkg/workflows/datasync/activities/benthos-builder.go
@@ -546,7 +546,7 @@ func (b *benthosBuilder) GenerateBenthosConfigs(
 
 				resp.Config.Output.Broker.Outputs = append(resp.Config.Output.Broker.Outputs, neosync_benthos.Outputs{
 					AwsS3: &neosync_benthos.AwsS3Insert{
-						Bucket:      connection.AwsS3Config.BucketArn,
+						Bucket:      connection.AwsS3Config.Bucket,
 						MaxInFlight: 64,
 						Path:        fmt.Sprintf("/%s", strings.Join(s3pathpieces, "/")),
 						Batching: &neosync_benthos.Batching{