Skip to content

Commit

Permalink
[Fix] [Connectors-v2-file-ftp] add ftp sink root path test case (apac…
Browse files Browse the repository at this point in the history
  • Loading branch information
xumingbei committed Dec 28, 2023
1 parent 52f2a8e commit 38b4874
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ public void startUp() throws Exception {
"/home/vsftpd/seatunnel/tmp/seatunnel/read/excel_filter/name=tyrantlucifer/hobby=coding/e2e_filter.xlsx",
ftpContainer);

ContainerUtil.copyFileIntoContainers(
"/excel/e2e.xlsx", "/home/vsftpd/seatunnel/e2e.xlsx", ftpContainer);

ftpContainer.execInContainer("sh", "-c", "chmod -R 777 /home/vsftpd/seatunnel/");
ftpContainer.execInContainer("sh", "-c", "chown -R ftp:ftp /home/vsftpd/seatunnel/");
}
Expand Down Expand Up @@ -136,6 +139,8 @@ public void testFtpFileReadAndWrite(TestContainer container)
helper.execute("/parquet/fake_to_ftp_file_parquet.conf");
// test write ftp orc file
helper.execute("/orc/fake_to_ftp_file_orc.conf");
// test write ftp root path excel file
helper.execute("/excel/fake_source_to_ftp_root_path_excel.conf");
}

@AfterAll
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

env {
parallelism = 1
job.mode = "BATCH"

# You can set spark configuration here
spark.app.name = "SeaTunnel"
spark.executor.instances = 1
spark.executor.cores = 1
spark.executor.memory = "1g"
spark.master = local
}

source {
FakeSource {
result_table_name = "ftp"
schema = {
fields {
c_map = "map<string, string>"
c_array = "array<int>"
c_string = string
c_boolean = boolean
c_tinyint = tinyint
c_smallint = smallint
c_int = int
c_bigint = bigint
c_float = float
c_double = double
c_bytes = bytes
c_date = date
c_decimal = "decimal(38, 18)"
c_timestamp = timestamp
c_row = {
c_map = "map<string, string>"
c_array = "array<int>"
c_string = string
c_boolean = boolean
c_tinyint = tinyint
c_smallint = smallint
c_int = int
c_bigint = bigint
c_float = float
c_double = double
c_bytes = bytes
c_date = date
c_decimal = "decimal(38, 18)"
c_timestamp = timestamp
}
}
}
}
}

sink {
FtpFile {
host = "ftp"
port = 21
user = seatunnel
password = pass
path = "/"
source_table_name = "ftp"
partition_dir_expression = "${k0}=${v0}"
is_partition_field_write_in_file = true
file_name_expression = "${transactionId}_${now}"
file_format_type = "excel"
filename_time_format = "yyyy.MM.dd"
is_enable_transaction = true
}
}

0 comments on commit 38b4874

Please sign in to comment.