Skip to content

ci: transform test add copy from s3. #10581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 15, 2023
Merged

ci: transform test add copy from s3. #10581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_06_transform.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,7 @@ ERROR 1105 (HY000) at line 1: Code: 1002, displayText = stage table function onl
4 6
5 6
5 6
--- copy from s3
2
3
4
23 changes: 16 additions & 7 deletions tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_06_transform.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,32 @@ DATADIR="fs://$DATADIR_PATH/"
echo "copy into '${DATADIR}' from t1 FILE_FORMAT = (type = PARQUET);" | $MYSQL_CLIENT_CONNECT
touch ${DATADIR_PATH}/transform.csv

echo "drop stage if exists s2;" | $MYSQL_CLIENT_CONNECT
echo "create stage s2 url = '${DATADIR}' FILE_FORMAT = (type = TSV);" | $MYSQL_CLIENT_CONNECT
echo "drop stage if exists s1;" | $MYSQL_CLIENT_CONNECT
echo "create stage s1 url = '${DATADIR}' FILE_FORMAT = (type = TSV);" | $MYSQL_CLIENT_CONNECT

echo '--- copy 1'
echo "copy into t1 from (select (t.id+1), age from @s2 t) FILE_FORMAT = (type = parquet) PATTERN='.*parquet';" | $MYSQL_CLIENT_CONNECT
echo "copy into t1 from (select (t.id+1), age from @s1 t) FILE_FORMAT = (type = parquet) PATTERN='.*parquet';" | $MYSQL_CLIENT_CONNECT
echo "select * from t1 order by id;" | $MYSQL_CLIENT_CONNECT

echo '--- copy 2'
echo "copy into t1 from (select (t.id+1), age from @s2 t) FILE_FORMAT = (type = parquet) PATTERN='.*parquet';" | $MYSQL_CLIENT_CONNECT
echo "copy into t1 from (select (t.id+1), age from @s1 t) FILE_FORMAT = (type = parquet) PATTERN='.*parquet';" | $MYSQL_CLIENT_CONNECT
echo "select * from t1 order by id;" | $MYSQL_CLIENT_CONNECT

echo '--- copy 3'
echo "copy into t1 from (select (t.id+1), age from @s2 t) FILE_FORMAT = (type = parquet) PATTERN='.*parquet' force=true;" | $MYSQL_CLIENT_CONNECT
echo "copy into t1 from (select (t.id+1), age from @s1 t) FILE_FORMAT = (type = parquet) PATTERN='.*parquet' force=true;" | $MYSQL_CLIENT_CONNECT
echo "select * from t1 order by id;" | $MYSQL_CLIENT_CONNECT

echo '--- copy csv'
echo "copy into t1 from (select (t.id+1), age from @s2 t) FILE_FORMAT = (type = csv) PATTERN='.*csv' force=true;" | $MYSQL_CLIENT_CONNECT
echo "copy into t1 from (select (t.id+1), age from @s1 t) FILE_FORMAT = (type = csv) PATTERN='.*csv' force=true;" | $MYSQL_CLIENT_CONNECT
echo "select * from t1 order by id;" | $MYSQL_CLIENT_CONNECT

#rm -rf ${DATADIR_PATH}
echo '--- copy from s3'
echo "drop table if exists t2;" | $MYSQL_CLIENT_CONNECT
echo "CREATE TABLE t2 (a INT32);" | $MYSQL_CLIENT_CONNECT

echo "drop stage if exists s2;" | $MYSQL_CLIENT_CONNECT
echo "create stage s2 url='s3://testbucket/admin/data/' connection=(aws_key_id = 'minioadmin' aws_secret_key = 'minioadmin' endpoint_url = 'http://127.0.0.1:9900/') file_format = (type=parquet)" | $MYSQL_CLIENT_CONNECT
echo "copy into t2 from (select (t.id+1) from @s2 t) files=('tuple.parquet');" | $MYSQL_CLIENT_CONNECT
echo "select * from t2 order by a;" | $MYSQL_CLIENT_CONNECT

rm -rf ${DATADIR_PATH}