From 7ffc21ab3b955392bfd1778e8fcf8a3fb6489b66 Mon Sep 17 00:00:00 2001 From: Sankar Hariappan Date: Wed, 2 May 2018 12:18:39 +0530 Subject: [PATCH 1/5] HIVE-18193: Migrate existing ACID tables to use write id per table rather than global transaction id. --- .../sql/mysql/hive-schema-3.0.0.mysql.sql | 4 +-- .../mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index 8e55e94374f5..feaf79df4487 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -986,7 +986,7 @@ CREATE TABLE TXN_COMPONENTS ( TC_TABLE varchar(128) NOT NULL, TC_PARTITION varchar(767), TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint, + TC_WRITEID bigint NOT NULL, FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -998,7 +998,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE; diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index 9a48346c5ca5..b623dee31276 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -292,6 +292,31 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT db.NAME, tblName.TBL_NAME FROM DBS db, + (SELECT tbl.DB_ID, tbl.TBL_NAME FROM TBLS tbl, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') tblParam + WHERE tbl.TBL_ID=tblParam.TBL_ID) tblName + where db.DB_ID=tblName.DB_ID) dbTblName, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) nextTxn; + +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT db.NAME, tblName.TBL_NAME FROM DBS db, + (SELECT tbl.DB_ID, tbl.TBL_NAME FROM TBLS tbl, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') tblParam + WHERE tbl.TBL_ID=tblParam.TBL_ID) tblName + where db.DB_ID=tblName.DB_ID) dbTblName, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) nextTxn; + +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + +ALTER TABLE TXN_COMPONENTS CHANGE COLUMN TC_WRITEID TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS CHANGE COLUMN CTC_WRITEID CTC_WRITEID bigint NOT NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; From a1f24e1112a9a33056ccf0e23ba5402225bc4e05 Mon Sep 17 00:00:00 2001 From: Sankar Hariappan Date: Thu, 3 May 2018 14:53:06 +0530 Subject: [PATCH 2/5] Added NOT NULL constraint --- metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql | 4 ++-- .../org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java | 8 ++++---- .../src/main/sql/derby/hive-schema-3.0.0.derby.sql | 8 ++++---- .../src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 4 ++-- .../src/main/sql/mssql/hive-schema-3.0.0.mssql.sql | 8 ++++---- .../src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 4 ++-- .../src/main/sql/mysql/hive-schema-3.0.0.mysql.sql | 2 +- .../src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 7 ++----- .../src/main/sql/oracle/hive-schema-3.0.0.oracle.sql | 8 ++++---- .../src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 4 ++-- .../src/main/sql/postgres/hive-schema-3.0.0.postgres.sql | 8 ++++---- .../main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql | 4 ++-- 12 files changed, 33 insertions(+), 36 deletions(-) diff --git a/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql b/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql index 3c20d1ed8ab4..dbaaab75b288 100644 --- a/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql +++ b/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql @@ -18,8 +18,8 @@ CREATE TABLE NEXT_WRITE_ID ( CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE); -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- Modify Compaction related tables to use write id instead of txn id RENAME COLUMN COMPACTION_QUEUE.CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE_ID; diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index cf89ab2166b6..838be7d36e03 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -82,20 +82,20 @@ public static void prepDb(Configuration conf) throws Exception { " TXN_HOST varchar(128) NOT NULL)"); stmt.execute("CREATE TABLE TXN_COMPONENTS (" + - " TC_TXNID bigint REFERENCES TXNS (TXN_ID)," + + " TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID)," + " TC_DATABASE varchar(128) NOT NULL," + " TC_TABLE varchar(128)," + " TC_PARTITION varchar(767)," + " TC_OPERATION_TYPE char(1) NOT NULL," + - " TC_WRITEID bigint)"); + " TC_WRITEID bigint NOT NULL)"); stmt.execute("CREATE TABLE COMPLETED_TXN_COMPONENTS (" + - " CTC_TXNID bigint," + + " CTC_TXNID bigint NOT NULL," + " CTC_DATABASE varchar(128) NOT NULL," + " CTC_TABLE varchar(128)," + " CTC_PARTITION varchar(767)," + " CTC_ID bigint GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) NOT NULL," + " CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL," + - " CTC_WRITEID bigint)"); + " CTC_WRITEID bigint NOT NULL)"); stmt.execute("CREATE TABLE NEXT_TXN_ID (" + " NTXN_NEXT bigint NOT NULL)"); stmt.execute("INSERT INTO NEXT_TXN_ID VALUES(1)"); diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql index 8e097553c335..9854f597d083 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql @@ -507,23 +507,23 @@ CREATE TABLE TXNS ( ); CREATE TABLE TXN_COMPONENTS ( - TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID), TC_DATABASE varchar(128) NOT NULL, TC_TABLE varchar(128), TC_PARTITION varchar(767), TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint + TC_WRITEID bigint NOT NULL ); CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID); CREATE TABLE COMPLETED_TXN_COMPONENTS ( - CTC_TXNID bigint, + CTC_TXNID bigint NOT NULL, CTC_DATABASE varchar(128) NOT NULL, CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql index 73bef36d684f..daedd7fcda4b 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -154,8 +154,8 @@ RENAME COLUMN COMPACTION_QUEUE.CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE_ID; RENAME COLUMN COMPLETED_COMPACTIONS.CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_ID; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql index 51df92cff1e9..3f60364a642a 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql @@ -1022,12 +1022,12 @@ PRIMARY KEY CLUSTERED ); CREATE TABLE COMPLETED_TXN_COMPONENTS( - CTC_TXNID bigint NULL, + CTC_TXNID bigint NOT NULL, CTC_DATABASE nvarchar(128) NOT NULL, CTC_TABLE nvarchar(128) NULL, CTC_PARTITION nvarchar(767) NULL, CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1091,12 +1091,12 @@ PRIMARY KEY CLUSTERED ); CREATE TABLE TXN_COMPONENTS( - TC_TXNID bigint NULL, + TC_TXNID bigint NOT NULL, TC_DATABASE nvarchar(128) NOT NULL, TC_TABLE nvarchar(128) NULL, TC_PARTITION nvarchar(767) NULL, TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint + TC_WRITEID bigint NOT NULL ); ALTER TABLE TXN_COMPONENTS WITH CHECK ADD FOREIGN KEY(TC_TXNID) REFERENCES TXNS (TXN_ID); diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index a7232dd443e3..dcba5894f7fe 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -206,8 +206,8 @@ EXEC SP_RENAME 'COMPACTION_QUEUE.CQ_HIGHEST_TXN_ID', 'CQ_HIGHEST_WRITE_ID', 'COL EXEC SP_RENAME 'COMPLETED_COMPACTIONS.CC_HIGHEST_TXN_ID', 'CC_HIGHEST_WRITE_ID', 'COLUMN'; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index feaf79df4487..316e574ca0e3 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -983,7 +983,7 @@ CREATE TABLE TXNS ( CREATE TABLE TXN_COMPONENTS ( TC_TXNID bigint NOT NULL, TC_DATABASE varchar(128) NOT NULL, - TC_TABLE varchar(128) NOT NULL, + TC_TABLE varchar(128), TC_PARTITION varchar(767), TC_OPERATION_TYPE char(1) NOT NULL, TC_WRITEID bigint NOT NULL, diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index b623dee31276..4d526aab5306 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -194,8 +194,8 @@ ALTER TABLE COMPACTION_QUEUE CHANGE `CQ_HIGHEST_TXN_ID` `CQ_HIGHEST_WRITE_ID` bi ALTER TABLE COMPLETED_COMPACTIONS CHANGE `CC_HIGHEST_TXN_ID` `CC_HIGHEST_WRITE_ID` bigint; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint @@ -314,9 +314,6 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; -ALTER TABLE TXN_COMPONENTS CHANGE COLUMN TC_WRITEID TC_WRITEID bigint NOT NULL; -ALTER TABLE COMPLETED_TXN_COMPONENTS CHANGE COLUMN CTC_WRITEID CTC_WRITEID bigint NOT NULL; - -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index 3a12e087a601..42281a0d7394 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -958,23 +958,23 @@ CREATE TABLE TXNS ( ) ROWDEPENDENCIES; CREATE TABLE TXN_COMPONENTS ( - TC_TXNID NUMBER(19) REFERENCES TXNS (TXN_ID), + TC_TXNID NUMBER(19) NOT NULL REFERENCES TXNS (TXN_ID), TC_DATABASE VARCHAR2(128) NOT NULL, TC_TABLE VARCHAR2(128), TC_PARTITION VARCHAR2(767) NULL, TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID NUMBER(19) + TC_WRITEID NUMBER(19) NOT NULL ) ROWDEPENDENCIES; CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID); CREATE TABLE COMPLETED_TXN_COMPONENTS ( - CTC_TXNID NUMBER(19), + CTC_TXNID NUMBER(19) NOT NULL, CTC_DATABASE VARCHAR2(128) NOT NULL, CTC_TABLE VARCHAR2(256), CTC_PARTITION VARCHAR2(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID NUMBER(19) + CTC_WRITEID NUMBER(19) NOT NULL ) ROWDEPENDENCIES; CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index 3be7e6571349..1312dff1cb10 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -213,8 +213,8 @@ ALTER TABLE COMPACTION_QUEUE RENAME COLUMN CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE ALTER TABLE COMPLETED_COMPACTIONS RENAME COLUMN CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_ID; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID number(19); -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID number(19); +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID number(19) NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID number(19) NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql index 0152f48050a8..580649ff2f78 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql @@ -1645,23 +1645,23 @@ CREATE TABLE TXNS ( ); CREATE TABLE TXN_COMPONENTS ( - TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID), TC_DATABASE varchar(128) NOT NULL, TC_TABLE varchar(128), TC_PARTITION varchar(767) DEFAULT NULL, TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint + TC_WRITEID bigint NOT NULL ); CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS USING hash (TC_TXNID); CREATE TABLE COMPLETED_TXN_COMPONENTS ( - CTC_TXNID bigint, + CTC_TXNID bigint NOT NULL, CTC_DATABASE varchar(128) NOT NULL, CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS USING btree (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index fed8a93ed632..29031493aabc 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -231,8 +231,8 @@ ALTER TABLE COMPACTION_QUEUE RENAME CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE_ID; ALTER TABLE COMPLETED_COMPACTIONS RENAME CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_ID; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint From 5ee969a8ace98d7fb3c495338b3ac1aebe0e1045 Mon Sep 17 00:00:00 2001 From: Sankar Hariappan Date: Thu, 3 May 2018 15:20:13 +0530 Subject: [PATCH 3/5] Updated upgrade scripts of all databases for write ID migration --- .../upgrade/derby/057-HIVE-18193.derby.sql | 24 ++++++++++++++ .../derby/upgrade-2.3.0-to-3.0.0.derby.sql | 25 ++++++++++++++ .../mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 25 ++++++++++++++ .../mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 33 +++++++++++-------- .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 25 ++++++++++++++ .../upgrade-2.3.0-to-3.0.0.postgres.sql | 25 ++++++++++++++ 6 files changed, 143 insertions(+), 14 deletions(-) create mode 100644 metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql diff --git a/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql b/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql new file mode 100644 index 000000000000..946cce0ea105 --- /dev/null +++ b/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql @@ -0,0 +1,24 @@ + +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql index daedd7fcda4b..f3afbcfadc02 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -254,6 +254,31 @@ CREATE TABLE "APP"."RUNTIME_STATS" ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index dcba5894f7fe..d8009672d54e 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -321,6 +321,31 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index 4d526aab5306..568257e63ab5 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -293,27 +293,32 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); -- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) SELECT * FROM - (SELECT db.NAME, tblName.TBL_NAME FROM DBS db, - (SELECT tbl.DB_ID, tbl.TBL_NAME FROM TBLS tbl, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') tblParam - WHERE tbl.TBL_ID=tblParam.TBL_ID) tblName - where db.DB_ID=tblName.DB_ID) dbTblName, - (SELECT NTXN_NEXT FROM NEXT_TXN_ID) nextTxn; - + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) SELECT * FROM - (SELECT db.NAME, tblName.TBL_NAME FROM DBS db, - (SELECT tbl.DB_ID, tbl.TBL_NAME FROM TBLS tbl, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') tblParam - WHERE tbl.TBL_ID=tblParam.TBL_ID) tblName - where db.DB_ID=tblName.DB_ID) dbTblName, - (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) nextTxn; - + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; +ALTER TABLE TXN_COMPONENTS MODIFY COLUMN TC_TABLE varchar(128) NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index 1312dff1cb10..9cec2a74af90 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -310,6 +310,31 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index 29031493aabc..1ac987eb878c 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -329,6 +329,31 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.0.0', "VERSION_COMMENT"='Hive release version 3.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0'; From 11186197326856451c19752012b2a631e5caae1c Mon Sep 17 00:00:00 2001 From: Sankar Hariappan Date: Thu, 3 May 2018 19:16:58 +0530 Subject: [PATCH 4/5] Bug fixes with metastore install/upgrade tests --- .../derby/upgrade-2.3.0-to-3.0.0.derby.sql | 2 +- .../mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 2 +- .../mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 2 +- .../sql/oracle/hive-schema-3.0.0.oracle.sql | 18 +++++++------- .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 20 ++++++++-------- .../upgrade-2.3.0-to-3.0.0.postgres.sql | 24 +++++++++---------- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql index f3afbcfadc02..fdcdcc4ba075 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -263,7 +263,7 @@ INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, - (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_WRITE; -- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index d8009672d54e..e032616b6a41 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -330,7 +330,7 @@ INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, - (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_WRITE; -- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index 568257e63ab5..8ebb5f2d8d64 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -301,7 +301,7 @@ INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, - (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_WRITE; -- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index 42281a0d7394..736ee8e8ee94 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -1066,26 +1066,26 @@ CREATE TABLE WRITE_SET ( ); CREATE TABLE TXN_TO_WRITE_ID ( - T2W_TXNID number(19) NOT NULL, - T2W_DATABASE varchar(128) NOT NULL, - T2W_TABLE varchar(256) NOT NULL, - T2W_WRITEID number(19) NOT NULL + T2W_TXNID NUMBER(19) NOT NULL, + T2W_DATABASE VARCHAR2(128) NOT NULL, + T2W_TABLE VARCHAR2(256) NOT NULL, + T2W_WRITEID NUMBER(19) NOT NULL ); CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID); CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID); CREATE TABLE NEXT_WRITE_ID ( - NWI_DATABASE varchar(128) NOT NULL, - NWI_TABLE varchar(256) NOT NULL, - NWI_NEXT number(19) NOT NULL + NWI_DATABASE VARCHAR2(128) NOT NULL, + NWI_TABLE VARCHAR2(256) NOT NULL, + NWI_NEXT NUMBER(19) NOT NULL ); CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE); CREATE TABLE MIN_HISTORY_LEVEL ( - MHL_TXNID number(19) NOT NULL, - MHL_MIN_OPEN_TXNID number(19) NOT NULL, + MHL_TXNID NUMBER(19) NOT NULL, + MHL_MIN_OPEN_TXNID NUMBER(19) NOT NULL, PRIMARY KEY(MHL_TXNID) ); diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index 9cec2a74af90..dab2b1e22f76 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -191,19 +191,19 @@ UPDATE DBS -- HIVE-18192 CREATE TABLE TXN_TO_WRITE_ID ( - T2W_TXNID number(19) NOT NULL, - T2W_DATABASE varchar(128) NOT NULL, - T2W_TABLE varchar(256) NOT NULL, - T2W_WRITEID number(19) NOT NULL + T2W_TXNID NUMBER(19) NOT NULL, + T2W_DATABASE VARCHAR2(128) NOT NULL, + T2W_TABLE VARCHAR2(256) NOT NULL, + T2W_WRITEID NUMBER(19) NOT NULL ); CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID); CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID); CREATE TABLE NEXT_WRITE_ID ( - NWI_DATABASE varchar(128) NOT NULL, - NWI_TABLE varchar(256) NOT NULL, - NWI_NEXT number(19) NOT NULL + NWI_DATABASE VARCHAR2(128) NOT NULL, + NWI_TABLE VARCHAR2(256) NOT NULL, + NWI_NEXT NUMBER(19) NOT NULL ); CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE); @@ -316,17 +316,17 @@ INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) SELECT * FROM (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND to_char(PARAM_VALUE)='true') TBL_PARAM WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, - (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_WRITE; -- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) SELECT * FROM (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND to_char(PARAM_VALUE)='true') TBL_PARAM WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index 1ac987eb878c..ee5f5357c6fb 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -333,22 +333,22 @@ CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); -- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) SELECT * FROM - (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, - (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM - WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO - where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, - (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + (SELECT "DB"."NAME", "TBL_INFO"."TBL_NAME" FROM "DBS" "DB", + (SELECT "TBL"."DB_ID", "TBL"."TBL_NAME" FROM "TBLS" "TBL", + (SELECT "TBL_ID" FROM "TABLE_PARAMS" WHERE "PARAM_KEY"='transactional' AND "PARAM_VALUE"='true') "TBL_PARAM" + WHERE "TBL"."TBL_ID"="TBL_PARAM"."TBL_ID") "TBL_INFO" + where "DB"."DB_ID"="TBL_INFO"."DB_ID") "DB_TBL_NAME", + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) "NEXT_WRITE"; -- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) SELECT * FROM - (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, - (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM - WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO - where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, - (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + (SELECT "DB"."NAME", "TBL_INFO"."TBL_NAME" FROM "DBS" "DB", + (SELECT "TBL"."DB_ID", "TBL"."TBL_NAME" FROM "TBLS" "TBL", + (SELECT "TBL_ID" FROM "TABLE_PARAMS" WHERE "PARAM_KEY"='transactional' AND "PARAM_VALUE"='true') "TBL_PARAM" + WHERE "TBL"."TBL_ID"="TBL_PARAM"."TBL_ID") "TBL_INFO" + where "DB"."DB_ID"="TBL_INFO"."DB_ID") "DB_TBL_NAME", + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) "TXN_INFO"; -- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; From 58b8a6fa73a2b22661a13075f6ca98b465f7f57b Mon Sep 17 00:00:00 2001 From: Sankar Hariappan Date: Fri, 4 May 2018 16:33:45 +0530 Subject: [PATCH 5/5] Bug fix for ptest failures --- metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql | 4 ++-- metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql | 4 ++-- .../org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java | 4 ++-- .../src/main/sql/derby/hive-schema-3.0.0.derby.sql | 4 ++-- .../src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 8 ++++---- .../src/main/sql/mssql/hive-schema-3.0.0.mssql.sql | 4 ++-- .../src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 4 ++-- .../src/main/sql/mysql/hive-schema-3.0.0.mysql.sql | 4 ++-- .../src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 4 ++-- .../src/main/sql/oracle/hive-schema-3.0.0.oracle.sql | 4 ++-- .../src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 4 ++-- .../src/main/sql/postgres/hive-schema-3.0.0.postgres.sql | 4 ++-- .../main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql | 4 ++-- 13 files changed, 28 insertions(+), 28 deletions(-) diff --git a/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql b/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql index dbaaab75b288..3c20d1ed8ab4 100644 --- a/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql +++ b/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql @@ -18,8 +18,8 @@ CREATE TABLE NEXT_WRITE_ID ( CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE); -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; -- Modify Compaction related tables to use write id instead of txn id RENAME COLUMN COMPACTION_QUEUE.CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE_ID; diff --git a/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql b/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql index 946cce0ea105..499d06ed8fba 100644 --- a/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql +++ b/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql @@ -4,7 +4,7 @@ INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) SELECT * FROM (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND CAST(PARAM_VALUE AS VARCHAR(128))='true') TBL_PARAM WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; @@ -14,7 +14,7 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) SELECT * FROM (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND CAST(PARAM_VALUE AS VARCHAR(128))='true') TBL_PARAM WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index 838be7d36e03..45971663597f 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -87,7 +87,7 @@ public static void prepDb(Configuration conf) throws Exception { " TC_TABLE varchar(128)," + " TC_PARTITION varchar(767)," + " TC_OPERATION_TYPE char(1) NOT NULL," + - " TC_WRITEID bigint NOT NULL)"); + " TC_WRITEID bigint)"); stmt.execute("CREATE TABLE COMPLETED_TXN_COMPONENTS (" + " CTC_TXNID bigint NOT NULL," + " CTC_DATABASE varchar(128) NOT NULL," + @@ -95,7 +95,7 @@ public static void prepDb(Configuration conf) throws Exception { " CTC_PARTITION varchar(767)," + " CTC_ID bigint GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) NOT NULL," + " CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL," + - " CTC_WRITEID bigint NOT NULL)"); + " CTC_WRITEID bigint)"); stmt.execute("CREATE TABLE NEXT_TXN_ID (" + " NTXN_NEXT bigint NOT NULL)"); stmt.execute("INSERT INTO NEXT_TXN_ID VALUES(1)"); diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql index 9854f597d083..e818e1b7e2ac 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql @@ -512,7 +512,7 @@ CREATE TABLE TXN_COMPONENTS ( TC_TABLE varchar(128), TC_PARTITION varchar(767), TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint NOT NULL + TC_WRITEID bigint ); CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID); @@ -523,7 +523,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint NOT NULL + CTC_WRITEID bigint ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql index fdcdcc4ba075..7b7a8a24d9d0 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -154,8 +154,8 @@ RENAME COLUMN COMPACTION_QUEUE.CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE_ID; RENAME COLUMN COMPLETED_COMPACTIONS.CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_ID; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint @@ -260,7 +260,7 @@ INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) SELECT * FROM (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND CAST(PARAM_VALUE AS VARCHAR(128))='true') TBL_PARAM WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_WRITE; @@ -270,7 +270,7 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) SELECT * FROM (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, - (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND CAST(PARAM_VALUE AS VARCHAR(128))='true') TBL_PARAM WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql index 3f60364a642a..c88fb1897375 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql @@ -1027,7 +1027,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS( CTC_TABLE nvarchar(128) NULL, CTC_PARTITION nvarchar(767) NULL, CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint NOT NULL + CTC_WRITEID bigint ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1096,7 +1096,7 @@ CREATE TABLE TXN_COMPONENTS( TC_TABLE nvarchar(128) NULL, TC_PARTITION nvarchar(767) NULL, TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint NOT NULL + TC_WRITEID bigint ); ALTER TABLE TXN_COMPONENTS WITH CHECK ADD FOREIGN KEY(TC_TXNID) REFERENCES TXNS (TXN_ID); diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index e032616b6a41..14c3deb44cdb 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -206,8 +206,8 @@ EXEC SP_RENAME 'COMPACTION_QUEUE.CQ_HIGHEST_TXN_ID', 'CQ_HIGHEST_WRITE_ID', 'COL EXEC SP_RENAME 'COMPLETED_COMPACTIONS.CC_HIGHEST_TXN_ID', 'CC_HIGHEST_WRITE_ID', 'COLUMN'; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index 316e574ca0e3..c54df55ed92f 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -986,7 +986,7 @@ CREATE TABLE TXN_COMPONENTS ( TC_TABLE varchar(128), TC_PARTITION varchar(767), TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint NOT NULL, + TC_WRITEID bigint, FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -998,7 +998,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint NOT NULL + CTC_WRITEID bigint ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE; diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index 8ebb5f2d8d64..9b87563b8a98 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -194,8 +194,8 @@ ALTER TABLE COMPACTION_QUEUE CHANGE `CQ_HIGHEST_TXN_ID` `CQ_HIGHEST_WRITE_ID` bi ALTER TABLE COMPLETED_COMPACTIONS CHANGE `CC_HIGHEST_TXN_ID` `CC_HIGHEST_WRITE_ID` bigint; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index 736ee8e8ee94..63cc1f75e2fc 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -963,7 +963,7 @@ CREATE TABLE TXN_COMPONENTS ( TC_TABLE VARCHAR2(128), TC_PARTITION VARCHAR2(767) NULL, TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID NUMBER(19) NOT NULL + TC_WRITEID NUMBER(19) ) ROWDEPENDENCIES; CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID); @@ -974,7 +974,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE VARCHAR2(256), CTC_PARTITION VARCHAR2(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID NUMBER(19) NOT NULL + CTC_WRITEID NUMBER(19) ) ROWDEPENDENCIES; CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index dab2b1e22f76..ce3437f723fc 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -213,8 +213,8 @@ ALTER TABLE COMPACTION_QUEUE RENAME COLUMN CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE ALTER TABLE COMPLETED_COMPACTIONS RENAME COLUMN CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_ID; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID number(19) NOT NULL; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID number(19) NOT NULL; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID number(19); +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID number(19); -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql index 580649ff2f78..d210a55fcf4b 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql @@ -1650,7 +1650,7 @@ CREATE TABLE TXN_COMPONENTS ( TC_TABLE varchar(128), TC_PARTITION varchar(767) DEFAULT NULL, TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint NOT NULL + TC_WRITEID bigint ); CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS USING hash (TC_TXNID); @@ -1661,7 +1661,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint NOT NULL + CTC_WRITEID bigint ); CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS USING btree (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index ee5f5357c6fb..f2571d8f2b91 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -231,8 +231,8 @@ ALTER TABLE COMPACTION_QUEUE RENAME CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE_ID; ALTER TABLE COMPLETED_COMPACTIONS RENAME CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_ID; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint