Skip to content

[Test](regression-test): do not use path style to access s3 #35787

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hello-stephen merged 9 commits into from
Jul 13, 2024
Merged

[Test](regression-test): do not use path style to access s3 #35787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0a281be
[Test](regression-test): Normalize S3 http url (#35666)
Thearas May 31, 2024
f87b864
[Test](regression-test): do not use path style to access aliyun OSS (…
Thearas Jun 1, 2024
0ace3be
Merge branch 'branch-2.0' into 2.0-s3
Thearas Jun 3, 2024
61679a5
Merge branch 'branch-2.0' into 2.0-s3
xiaokang Jun 20, 2024
b850023
Merge branch 'branch-2.0' into 2.0-s3
Thearas Jul 10, 2024
9286c9c
fix s3 file name
Thearas Jul 11, 2024
2625450
fix s3 file name
Thearas Jul 11, 2024
66d2bec
Merge branch 'branch-2.0' into 2.0-s3
Thearas Jul 11, 2024
f3403fa
Merge branch 'branch-2.0' into 2.0-s3
Thearas Jul 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions regression-test/suites/export_p0/export/test_show_export.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ suite("test_show_export", "p0") {
def outfile_url = waiting_export.call(label)

order_qt_select_load1 """ select * from s3(
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.parquet",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.parquet",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "parquet",
Expand Down Expand Up @@ -154,7 +154,7 @@ suite("test_show_export", "p0") {
def outfile_url = waiting_export.call(label)

order_qt_select_load1 """ select * from s3(
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.parquet",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.parquet",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "parquet",
Expand Down
4 changes: 2 additions & 2 deletions regression-test/suites/export_p0/outfile/csv/test_outfile_empty_data.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,12 +152,12 @@ suite("test_outfile_empty_data", "external,hive,tvf,external_docker") {
"""

qt_select_tvf3 """ SELECT * FROM S3 (
"uri" = "http://${s3_endpoint}${outfile_to_s3_directly_url.substring(4, outfile_to_s3_directly_url.length())}0.csv",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_to_s3_directly_url.substring(5 + bucket.length(), outfile_to_s3_directly_url.length())}0.csv",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "${format}",
"region" = "${region}",
"use_path_style" = "true"
"use_path_style" = "false" -- aliyun does not support path_style
);
"""

Expand Down
4 changes: 2 additions & 2 deletions regression-test/suites/export_p0/test_outfile_file_suffix.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@ suite("test_outfile_file_suffix", "p0") {
def file_suffix = "txt";
def file_format = "csv";
def outfile_url = csv_suffix_result(file_suffix, file_format);
print("http://${s3_endpoint}${outfile_url.substring(4)}0.${file_suffix}")
print("http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.${file_suffix}")
qt_select """ select * from s3(
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.${file_suffix}",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.${file_suffix}",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "${file_format}",
Expand Down
8 changes: 4 additions & 4 deletions regression-test/suites/export_p0/test_with_bom.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ suite("test_with_bom", "p0") {
def outfile_url = waiting_export.call(label)

order_qt_select_load1 """ select * from s3(
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.${file_format}",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.${file_format}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "csv",
Expand Down Expand Up @@ -152,7 +152,7 @@ suite("test_with_bom", "p0") {
def outfile_url = waiting_export.call(label)

order_qt_select_load1 """ select * from s3(
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.${file_format}",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.${file_format}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "csv",
Expand Down Expand Up @@ -189,7 +189,7 @@ suite("test_with_bom", "p0") {
def outfile_url = waiting_export.call(label)

order_qt_select_load1 """ select * from s3(
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.${file_format}",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.${file_format}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "csv_with_names",
Expand Down Expand Up @@ -226,7 +226,7 @@ suite("test_with_bom", "p0") {
def outfile_url = waiting_export.call(label)

order_qt_select_load1 """ select * from s3(
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.${file_format}",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.${file_format}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "csv_with_names_and_types",
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/export_p2/test_export_with_s3.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ suite("test_export_with_s3", "p2") {

// check data correctness
order_qt_select """ select * from s3(
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.${file_suffix}",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length())}0.${file_suffix}",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"column_separator"=",",
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ suite("test_insert_from_tvf_with_common_user","p0,external,hive,tvf,external_doc

sql """ INSERT INTO ${load_table_name}
SELECT * FROM S3 (
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.csv",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.csv",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "csv",
Expand Down
6 changes: 3 additions & 3 deletions regression-test/suites/external_table_p0/tvf/test_s3_tvf.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ suite("test_s3_tvf", "p0") {
// 1. normal
try {
order_qt_select_1 """ SELECT * FROM S3 (
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.orc",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.orc",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "orc",
Expand Down Expand Up @@ -107,11 +107,11 @@ suite("test_s3_tvf", "p0") {
// 3.test use_path_style
try {
order_qt_select_3 """ SELECT * FROM S3 (
"uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.orc",
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length())}0.orc",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "orc",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"region" = "${region}"
);
"""
Expand Down
20 changes: 10 additions & 10 deletions regression-test/suites/external_table_p0/tvf/test_s3_tvf_compression.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ suite("test_s3_tvf_compression", "p0") {
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator" = ",",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"compress_type" ="${compress_type}") order by c1,c2,c3,c4,c5 limit 20;
"""

Expand All @@ -49,7 +49,7 @@ suite("test_s3_tvf_compression", "p0") {
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator" = ",",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"compress_type" ="${compress_type}") order by cast(c1 as int),c4 limit 20;
"""

Expand All @@ -65,7 +65,7 @@ suite("test_s3_tvf_compression", "p0") {
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator" = ",",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"compress_type" ="${compress_type}") order by c1,c2,c3,c4,c5 limit 15;
"""

Expand All @@ -79,7 +79,7 @@ suite("test_s3_tvf_compression", "p0") {
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator" = ",",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"compress_type" ="${compress_type}") where c1!="100" order by cast(c4 as date),c1 limit 13;
"""

Expand All @@ -95,7 +95,7 @@ suite("test_s3_tvf_compression", "p0") {
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator" = ",",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"compress_type" ="${compress_type}FRAME") order by c1,c2,c3,c4,c5 limit 14;
"""

Expand All @@ -109,7 +109,7 @@ suite("test_s3_tvf_compression", "p0") {
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator" = ",",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"compress_type" ="${compress_type}FRAME") where c3="buHDwfGeNHfpRFdNaogneddi" order by c3,c1 limit 14;
"""

Expand All @@ -126,7 +126,7 @@ suite("test_s3_tvf_compression", "p0") {
"s3.secret_key" = "${sk}",
"REGION" = "${region}",
"FORMAT" = "csv",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"column_separator" = '\001',
"compress_type" ="${compress_type}") ${orderBy_limit};
"""
Expand All @@ -140,7 +140,7 @@ suite("test_s3_tvf_compression", "p0") {
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator" = '\001',
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"compress_type" ="${compress_type}") group by c1,c2 order by c1,c2 limit 5;
"""

Expand All @@ -156,7 +156,7 @@ suite("test_s3_tvf_compression", "p0") {
"s3.secret_key" = "${sk}",
"REGION" = "${region}",
"FORMAT" = "csv",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"column_separator" = '\001',
"compress_type" ="${compress_type}block") ${orderBy_limit};
"""
Expand All @@ -170,7 +170,7 @@ suite("test_s3_tvf_compression", "p0") {
"s3.secret_key" = "${sk}",
"REGION" = "${region}",
"FORMAT" = "csv",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"column_separator" = '\001',
"compress_type" ="${compress_type}block") where c2 ="abccc";
"""
Expand Down
16 changes: 8 additions & 8 deletions regression-test/suites/external_table_p2/tvf/test_path_partition_keys.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,8 @@ suite("test_path_partition_keys", "p2,external,tvf,external_remote,external_remo
"s3.secret_key" = "${sk}",
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator"="${column_separator}",
"use_path_style" = "true",
"column_separator" = ",",
"use_path_style" = "false", -- aliyun does not support path_style
"path_partition_keys"="dt1")
"""

Expand All @@ -152,8 +152,8 @@ suite("test_path_partition_keys", "p2,external,tvf,external_remote,external_remo
"s3.secret_key" = "${sk}",
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator"="${column_separator}",
"use_path_style" = "true",
"column_separator" = ",",
"use_path_style" = "false", -- aliyun does not support path_style
"path_partition_keys"="dt1") limit 3;
"""

Expand All @@ -166,8 +166,8 @@ suite("test_path_partition_keys", "p2,external,tvf,external_remote,external_remo
"s3.secret_key" = "${sk}",
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator"="${column_separator}",
"use_path_style" = "true",
"column_separator" = ",",
"use_path_style" = "false", -- aliyun does not support path_style
"path_partition_keys"="dt1") limit 3;
"""

Expand All @@ -180,8 +180,8 @@ suite("test_path_partition_keys", "p2,external,tvf,external_remote,external_remo
"s3.secret_key" = "${sk}",
"REGION" = "${region}",
"FORMAT" = "csv",
"column_separator"="${column_separator}",
"use_path_style" = "true",
"use_path_style" = "false", -- aliyun does not support path_style
"column_separator" = ",",
"path_partition_keys"="dt2,dt1") limit 3;
"""
}
8 changes: 4 additions & 4 deletions regression-test/suites/load_p0/tvf/test_tvf_error_url.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ suite("test_tvf_error_url", "p0") {

String path = "select_tvf/no_exists_file_test"
order_qt_select """ SELECT * FROM S3 (
"uri" = "http://${s3_endpoint}/${bucket}/${path}/no_exist_file1.csv",
"uri" = "http://${bucket}.${s3_endpoint}/${path}/no_exist_file1.csv",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "csv",
Expand All @@ -33,7 +33,7 @@ suite("test_tvf_error_url", "p0") {
"""

order_qt_desc """ desc function S3 (
"uri" = "http://${s3_endpoint}/${bucket}/${path}/no_exist_file1.csv",
"uri" = "http://${bucket}.${s3_endpoint}/${path}/no_exist_file1.csv",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "csv",
Expand All @@ -42,7 +42,7 @@ suite("test_tvf_error_url", "p0") {
"""

order_qt_select2 """ SELECT * FROM S3 (
"uri" = "http://${s3_endpoint}/${bucket}/${path}/*.csv",
"uri" = "http://${bucket}.${s3_endpoint}/${path}/*.csv",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "csv",
Expand All @@ -51,7 +51,7 @@ suite("test_tvf_error_url", "p0") {
"""

order_qt_desc2 """ desc function S3 (
"uri" = "http://${s3_endpoint}/${bucket}/${path}/*.csv",
"uri" = "http://${bucket}.${s3_endpoint}/${path}/*.csv",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "csv",
Expand Down