Skip to content

Commit

Permalink
[KYUUBI #3967] Shorten column varchar length of metadata table
Browse files Browse the repository at this point in the history 
### _Why are the changes needed?_

`Specified key was too long; max key length is 3072 bytes`

Currently, the table uses utf8mb4, so the allowed varchar column to create index must be shorter than `varchar(768)`.

```
2022-11-28 21:06:47 [ERROR] [main] org.apache.kyuubi.Utils#77 - Uncaught exception in thread main
org.apache.kyuubi.KyuubiException: Error executing -- the metadata table ddl

CREATE TABLE metadata(
    key_id bigint PRIMARY KEY AUTO_INCREMENT COMMENT 'the auto increment key id',
    identifier varchar(36) NOT NULL COMMENT 'the identifier id, which is an UUID',
    session_type varchar(128) NOT NULL COMMENT 'the session type, SQL or BATCH',
    real_user varchar(1024) NOT NULL COMMENT 'the real user',
    user_name varchar(1024) NOT NULL COMMENT 'the user name, might be a proxy user',
    ip_address varchar(512) COMMENT 'the client ip address',
    kyuubi_instance varchar(1024) NOT NULL COMMENT 'the kyuubi instance that creates this',
    state varchar(128) NOT NULL COMMENT 'the session state',
    resource varchar(1024) COMMENT 'the main resource',
    class_name varchar(1024) COMMENT 'the main class name',
    request_name varchar(1024) COMMENT 'the request name',
    request_conf mediumtext COMMENT 'the request config map',
    request_args mediumtext COMMENT 'the request arguments',
    create_time BIGINT NOT NULL COMMENT 'the metadata create time',
    engine_type varchar(1024) NOT NULL COMMENT 'the engine type',
    cluster_manager varchar(128) COMMENT 'the engine cluster manager',
    engine_id varchar(128) COMMENT 'the engine application id',
    engine_name mediumtext COMMENT 'the engine application name',
    engine_url varchar(1024) COMMENT 'the engine tracking url',
    engine_state varchar(128) COMMENT 'the engine application state',
    engine_error mediumtext COMMENT 'the engine application diagnose',
    end_time bigint COMMENT 'the metadata end time',
    peer_instance_closed boolean default '0' COMMENT 'closed by peer kyuubi instance',
    INDEX kyuubi_instance_index(kyuubi_instance),
    UNIQUE INDEX unique_identifier_index(identifier),
    INDEX user_name_index(user_name),
    INDEX engine_type_index(engine_type)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4:Specified key was too long; max key length is 3072 bytes
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.execute(JDBCMetadataStore.scala:372) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.$anonfun$initSchema$4(JDBCMetadataStore.scala:95) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.$anonfun$initSchema$4$adapted(JDBCMetadataStore.scala:94) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) ~[scala-library-2.12.17.jar:?]
 at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) ~[scala-library-2.12.17.jar:?]
 at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198) ~[scala-library-2.12.17.jar:?]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.$anonfun$initSchema$3(JDBCMetadataStore.scala:94) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.Utils$.tryLogNonFatalError(Utils.scala:245) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.$anonfun$initSchema$2(JDBCMetadataStore.scala:94) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.$anonfun$initSchema$2$adapted(JDBCMetadataStore.scala:92) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.withConnection(JDBCMetadataStore.scala:445) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.$anonfun$initSchema$1(JDBCMetadataStore.scala:92) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.$anonfun$initSchema$1$adapted(JDBCMetadataStore.scala:88) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.17.jar:?]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.initSchema(JDBCMetadataStore.scala:88) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.<init>(JDBCMetadataStore.scala:76) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_352]
 at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_352]
 at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_352]
 at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_352]
 at org.apache.kyuubi.util.ClassUtils$.createInstance(ClassUtils.scala:45) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.MetadataManager$.createMetadataStore(MetadataManager.scala:295) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.metadata.MetadataManager.initialize(MetadataManager.scala:58) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1(CompositeService.scala:40) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1$adapted(CompositeService.scala:40) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.17.jar:?]
 at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.17.jar:?]
 at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.17.jar:?]
 at org.apache.kyuubi.service.CompositeService.initialize(CompositeService.scala:40) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.session.SessionManager.initialize(SessionManager.scala:268) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.session.KyuubiSessionManager.initialize(KyuubiSessionManager.scala:67) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1(CompositeService.scala:40) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1$adapted(CompositeService.scala:40) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.17.jar:?]
 at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.17.jar:?]
 at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.17.jar:?]
 at org.apache.kyuubi.service.CompositeService.initialize(CompositeService.scala:40) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.service.AbstractBackendService.initialize(AbstractBackendService.scala:209) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1(CompositeService.scala:40) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1$adapted(CompositeService.scala:40) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.17.jar:?]
 at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.17.jar:?]
 at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.17.jar:?]
 at org.apache.kyuubi.service.CompositeService.initialize(CompositeService.scala:40) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.service.Serverable.initialize(Serverable.scala:46) ~[kyuubi-common_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.KyuubiServer.initialize(KyuubiServer.scala:136) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.KyuubiServer$.startServer(KyuubiServer.scala:56) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.KyuubiServer$.main(KyuubiServer.scala:93) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 at org.apache.kyuubi.server.KyuubiServer.main(KyuubiServer.scala) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
Caused by: java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes
 at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:120) ~[mysql-connector-j-8.0.31.jar:8.0.31]
 at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:122) ~[mysql-connector-j-8.0.31.jar:8.0.31]
 at com.mysql.cj.jdbc.ClientPreparedStatement.executeInternal(ClientPreparedStatement.java:916) ~[mysql-connector-j-8.0.31.jar:8.0.31]
 at com.mysql.cj.jdbc.ClientPreparedStatement.execute(ClientPreparedStatement.java:354) ~[mysql-connector-j-8.0.31.jar:8.0.31]
 at com.zaxxer.hikari.pool.ProxyPreparedStatement.execute(ProxyPreparedStatement.java:44) ~[HikariCP-4.0.3.jar:?]
 at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.execute(HikariProxyPreparedStatement.java) ~[HikariCP-4.0.3.jar:?]
 at org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore.execute(JDBCMetadataStore.scala:369) ~[kyuubi-server_2.12-1.7.0-SNAPSHOT.jar:1.7.0-SNAPSHOT]
 ... 48 more
```

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #3967 from pan3793/metadata-sql.

Closes #3967

c60c911 [Cheng Pan] remove index on kyuubi_instance
878d50c [Cheng Pan] Shorten column varchar length of metadata table

Authored-by: Cheng Pan <chengpan@apache.org>
Signed-off-by: Cheng Pan <chengpan@apache.org>
  • Loading branch information
@pan3793
pan3793 committed Dec 12, 2022
1 parent fa9e6be commit b8fe4d9
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 15 deletions.
14 changes: 6 additions & 8 deletions kyuubi-server/src/main/resources/sql/derby/metadata-store-schema-derby.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
CREATE TABLE metadata(
key_id bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, -- the auto increment key id
identifier varchar(36) NOT NULL, -- the identifier id, which is an UUID
session_type varchar(128) NOT NULL, -- the session type, SQL or BATCH
real_user varchar(1024) NOT NULL, -- the real user
user_name varchar(1024) NOT NULL, -- the user name, might be a proxy user
ip_address varchar(512), -- the client ip address
session_type varchar(32) NOT NULL, -- the session type, SQL or BATCH
real_user varchar(255) NOT NULL, -- the real user
user_name varchar(255) NOT NULL, -- the user name, might be a proxy user
ip_address varchar(128), -- the client ip address
kyuubi_instance varchar(1024) NOT NULL, -- the kyuubi instance that creates this
state varchar(128) NOT NULL, -- the session state
resource varchar(1024), -- the main resource
Expand All @@ -17,19 +17,17 @@ CREATE TABLE metadata(
request_conf clob, -- the request config map
request_args clob, -- the request arguments
create_time BIGINT NOT NULL, -- the metadata create time
engine_type varchar(1024) NOT NULL, -- the engine type
engine_type varchar(32) NOT NULL, -- the engine type
cluster_manager varchar(128), -- the engine cluster manager
engine_id varchar(128), -- the engine application id
engine_name clob, -- the engine application name
engine_url varchar(1024), -- the engine tracking url
engine_state varchar(128), -- the engine application state
engine_state varchar(32), -- the engine application state
engine_error clob, -- the engine application diagnose
end_time bigint, -- the metadata end time
peer_instance_closed boolean default FALSE -- closed by peer kyuubi instance
);

CREATE INDEX metadata_kyuubi_instance_index ON metadata(kyuubi_instance);

CREATE UNIQUE INDEX metadata_unique_identifier_index ON metadata(identifier);

CREATE INDEX metadata_user_name_index ON metadata(user_name);
Expand Down
13 changes: 6 additions & 7 deletions kyuubi-server/src/main/resources/sql/mysql/metadata-store-schema-mysql.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
CREATE TABLE IF NOT EXISTS metadata(
key_id bigint PRIMARY KEY AUTO_INCREMENT COMMENT 'the auto increment key id',
identifier varchar(36) NOT NULL COMMENT 'the identifier id, which is an UUID',
session_type varchar(128) NOT NULL COMMENT 'the session type, SQL or BATCH',
real_user varchar(1024) NOT NULL COMMENT 'the real user',
user_name varchar(1024) NOT NULL COMMENT 'the user name, might be a proxy user',
ip_address varchar(512) COMMENT 'the client ip address',
session_type varchar(32) NOT NULL COMMENT 'the session type, SQL or BATCH',
real_user varchar(255) NOT NULL COMMENT 'the real user',
user_name varchar(255) NOT NULL COMMENT 'the user name, might be a proxy user',
ip_address varchar(128) COMMENT 'the client ip address',
kyuubi_instance varchar(1024) NOT NULL COMMENT 'the kyuubi instance that creates this',
state varchar(128) NOT NULL COMMENT 'the session state',
resource varchar(1024) COMMENT 'the main resource',
Expand All @@ -15,16 +15,15 @@ CREATE TABLE IF NOT EXISTS metadata(
request_conf mediumtext COMMENT 'the request config map',
request_args mediumtext COMMENT 'the request arguments',
create_time BIGINT NOT NULL COMMENT 'the metadata create time',
engine_type varchar(1024) NOT NULL COMMENT 'the engine type',
engine_type varchar(32) NOT NULL COMMENT 'the engine type',
cluster_manager varchar(128) COMMENT 'the engine cluster manager',
engine_id varchar(128) COMMENT 'the engine application id',
engine_name mediumtext COMMENT 'the engine application name',
engine_url varchar(1024) COMMENT 'the engine tracking url',
engine_state varchar(128) COMMENT 'the engine application state',
engine_state varchar(32) COMMENT 'the engine application state',
engine_error mediumtext COMMENT 'the engine application diagnose',
end_time bigint COMMENT 'the metadata end time',
peer_instance_closed boolean default '0' COMMENT 'closed by peer kyuubi instance',
INDEX kyuubi_instance_index(kyuubi_instance),
UNIQUE INDEX unique_identifier_index(identifier),
INDEX user_name_index(user_name),
INDEX engine_type_index(engine_type)
Expand Down

0 comments on commit b8fe4d9

Please sign in to comment.