HIVE-29643: Add missing primary keys to backend tables across differe…

standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.3.0.derby.sql

@@ -536,6 +536,7 @@ INSERT INTO TXNS (TXN_ID, TXN_STATE, TXN_STARTED, TXN_LAST_HEARTBEAT, TXN_USER,
   VALUES(0, 'c', 0, 0, '', '');
 
 CREATE TABLE TXN_COMPONENTS (
+  TC_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID),
   TC_DATABASE varchar(128) NOT NULL,
   TC_TABLE varchar(256),
@@ -547,6 +548,7 @@ CREATE TABLE TXN_COMPONENTS (
 CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID);
 
 CREATE TABLE COMPLETED_TXN_COMPONENTS (
+  CTC_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   CTC_TXNID bigint NOT NULL,
   CTC_DATABASE varchar(128) NOT NULL,
   CTC_TABLE varchar(256),
@@ -559,9 +561,10 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
 CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
 
 CREATE TABLE TXN_LOCK_TBL (
+  TLT_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   TXN_LOCK bigint NOT NULL
 );
-INSERT INTO TXN_LOCK_TBL VALUES(1);
+INSERT INTO TXN_LOCK_TBL (TXN_LOCK) VALUES(1);
 
 CREATE TABLE HIVE_LOCKS (
   HL_LOCK_EXT_ID bigint NOT NULL,
@@ -587,9 +590,10 @@ CREATE TABLE HIVE_LOCKS (
 CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID);
 
 CREATE TABLE NEXT_LOCK_ID (
+  NL_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   NL_NEXT bigint NOT NULL
 );
-INSERT INTO NEXT_LOCK_ID VALUES(1);
+INSERT INTO NEXT_LOCK_ID (NL_NEXT) VALUES(1);
 
 CREATE TABLE COMPACTION_QUEUE (
   CQ_ID bigint PRIMARY KEY,
@@ -621,9 +625,10 @@ CREATE TABLE COMPACTION_QUEUE (
 );
 
 CREATE TABLE NEXT_COMPACTION_QUEUE_ID (
+  NCQI_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   NCQ_NEXT bigint NOT NULL
 );
-INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
+INSERT INTO NEXT_COMPACTION_QUEUE_ID (NCQ_NEXT) VALUES(1);
 
 CREATE TABLE COMPLETED_COMPACTIONS (
   CC_ID bigint PRIMARY KEY,
@@ -657,6 +662,7 @@ CREATE INDEX COMPLETED_COMPACTIONS_RES ON COMPLETED_COMPACTIONS (CC_DATABASE,CC_
 
 -- HIVE-25842
 CREATE TABLE COMPACTION_METRICS_CACHE (
+  CMC_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   CMC_DATABASE varchar(128) NOT NULL,
   CMC_TABLE varchar(256) NOT NULL,
   CMC_PARTITION varchar(767),
@@ -672,9 +678,8 @@ CREATE TABLE AUX_TABLE (
   PRIMARY KEY(MT_KEY1, MT_KEY2)
 );
 
---1st 4 cols make up a PK but since WS_PARTITION is nullable we can't declare such PK
---This is a good candidate for Index orgainzed table
 CREATE TABLE WRITE_SET (
+  WS_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   WS_DATABASE varchar(128) NOT NULL,
   WS_TABLE varchar(256) NOT NULL,
   WS_PARTITION varchar(767),
@@ -687,25 +692,25 @@ CREATE TABLE TXN_TO_WRITE_ID (
   T2W_TXNID bigint NOT NULL,
   T2W_DATABASE varchar(128) NOT NULL,
   T2W_TABLE varchar(256) NOT NULL,
-  T2W_WRITEID bigint NOT NULL
+  T2W_WRITEID bigint NOT NULL,
+  PRIMARY KEY (T2W_DATABASE, T2W_TABLE, T2W_TXNID)
 );
 
-CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID);
 CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID);
 
 CREATE TABLE NEXT_WRITE_ID (
   NWI_DATABASE varchar(128) NOT NULL,
   NWI_TABLE varchar(256) NOT NULL,
-  NWI_NEXT bigint NOT NULL
+  NWI_NEXT bigint NOT NULL,
+  PRIMARY KEY (NWI_DATABASE, NWI_TABLE)
 );
 
-CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE);
-
 CREATE TABLE MIN_HISTORY_WRITE_ID (
   MH_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID),
   MH_DATABASE varchar(128) NOT NULL,
   MH_TABLE varchar(256) NOT NULL,
-  MH_WRITEID bigint NOT NULL
+  MH_WRITEID bigint NOT NULL,
+  PRIMARY KEY (MH_TXNID, MH_DATABASE, MH_TABLE, MH_WRITEID)
 );
 
 CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);

standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.2.0-to-4.3.0.derby.sql

@@ -3,5 +3,22 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD COLUMN MRL_CAT_NAME varchar(128) N
 
 CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);
 
+ALTER TABLE TXN_COMPONENTS ADD COLUMN TC_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+ALTER TABLE COMPLETED_TXN_COMPONENTS ADD COLUMN CTC_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+ALTER TABLE COMPACTION_METRICS_CACHE ADD COLUMN CMC_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+ALTER TABLE WRITE_SET ADD COLUMN WS_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+
+DROP INDEX TBL_TO_TXN_ID_IDX;
+ALTER TABLE TXN_TO_WRITE_ID ADD PRIMARY KEY (T2W_DATABASE, T2W_TABLE, T2W_TXNID);
+
+DROP INDEX NEXT_WRITE_ID_IDX;
+ALTER TABLE NEXT_WRITE_ID ADD PRIMARY KEY (NWI_DATABASE, NWI_TABLE);
+
+ALTER TABLE MIN_HISTORY_WRITE_ID ADD PRIMARY KEY (MH_TXNID, MH_DATABASE, MH_TABLE, MH_WRITEID);
+
+ALTER TABLE TXN_LOCK_TBL ADD COLUMN TLT_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+ALTER TABLE NEXT_LOCK_ID ADD COLUMN NL_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+ALTER TABLE NEXT_COMPACTION_QUEUE_ID ADD COLUMN NCQI_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+
 -- This needs to be the last thing done.  Insert any changes above this line.
 UPDATE "APP".VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1;

standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.3.0.mssql.sql

@@ -1033,22 +1033,26 @@ CREATE INDEX COMPLETED_COMPACTIONS_RES ON COMPLETED_COMPACTIONS (CC_DATABASE,CC_
 
 -- HIVE-25842
 CREATE TABLE COMPACTION_METRICS_CACHE (
+    CMC_ID bigint NOT NULL IDENTITY(1,1),
     CMC_DATABASE nvarchar(128) NOT NULL,
     CMC_TABLE nvarchar(256) NOT NULL,
     CMC_PARTITION nvarchar(767) NULL,
     CMC_METRIC_TYPE nvarchar(128) NOT NULL,
     CMC_METRIC_VALUE int NOT NULL,
-    CMC_VERSION int NOT NULL
+    CMC_VERSION int NOT NULL,
+    PRIMARY KEY CLUSTERED (CMC_ID ASC)
 );
 
 CREATE TABLE COMPLETED_TXN_COMPONENTS(
+	CTC_ID bigint NOT NULL IDENTITY(1,1),
 	CTC_TXNID bigint NOT NULL,
 	CTC_DATABASE nvarchar(128) NOT NULL,
 	CTC_TABLE nvarchar(256) NULL,
 	CTC_PARTITION nvarchar(767) NULL,
     CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL,
     CTC_WRITEID bigint,
-    CTC_UPDATE_DELETE char(1) NOT NULL
+    CTC_UPDATE_DELETE char(1) NOT NULL,
+    PRIMARY KEY CLUSTERED (CTC_ID ASC)
 );
 
 
@@ -1080,22 +1084,28 @@ PRIMARY KEY CLUSTERED
 );
 
 CREATE TABLE NEXT_COMPACTION_QUEUE_ID(
-	NCQ_NEXT bigint NOT NULL
+	NCQI_ID bigint NOT NULL IDENTITY(1,1),
+	NCQ_NEXT bigint NOT NULL,
+	PRIMARY KEY CLUSTERED (NCQI_ID ASC)
 );
 
-INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
+INSERT INTO NEXT_COMPACTION_QUEUE_ID (NCQ_NEXT) VALUES(1);
 
 CREATE TABLE NEXT_LOCK_ID(
-	NL_NEXT bigint NOT NULL
+	NL_ID bigint NOT NULL IDENTITY(1,1),
+	NL_NEXT bigint NOT NULL,
+	PRIMARY KEY CLUSTERED (NL_ID ASC)
 );
 
-INSERT INTO NEXT_LOCK_ID VALUES(1);
+INSERT INTO NEXT_LOCK_ID (NL_NEXT) VALUES(1);
 
 CREATE TABLE TXN_LOCK_TBL(
-    TXN_LOCK bigint NOT NULL
+    TLT_ID bigint NOT NULL IDENTITY(1,1),
+    TXN_LOCK bigint NOT NULL,
+    PRIMARY KEY CLUSTERED (TLT_ID ASC)
 );
 
-INSERT INTO TXN_LOCK_TBL VALUES(1);
+INSERT INTO TXN_LOCK_TBL (TXN_LOCK) VALUES(1);
 
 CREATE TABLE TXNS(
     TXN_ID bigint NOT NULL IDENTITY(1,1),
@@ -1118,12 +1128,14 @@ INSERT INTO TXNS (TXN_ID, TXN_STATE, TXN_STARTED, TXN_LAST_HEARTBEAT, TXN_USER,
   VALUES(0, 'c', 0, 0, '', '');
 
 CREATE TABLE TXN_COMPONENTS(
+	TC_ID bigint NOT NULL IDENTITY(1,1),
 	TC_TXNID bigint NOT NULL,
 	TC_DATABASE nvarchar(128) NOT NULL,
 	TC_TABLE nvarchar(256) NULL,
 	TC_PARTITION nvarchar(767) NULL,
     TC_OPERATION_TYPE char(1) NOT NULL,
-    TC_WRITEID bigint
+    TC_WRITEID bigint,
+    PRIMARY KEY CLUSTERED (TC_ID ASC)
 );
 
 ALTER TABLE TXN_COMPONENTS  WITH CHECK ADD FOREIGN KEY(TC_TXNID) REFERENCES TXNS (TXN_ID);
@@ -1165,12 +1177,14 @@ CREATE INDEX CONSTRAINTS_PARENT_TBL_ID__INDEX ON KEY_CONSTRAINTS(PARENT_TBL_ID);
 CREATE INDEX CONSTRAINTS_CONSTRAINT_TYPE_INDEX ON KEY_CONSTRAINTS(CONSTRAINT_TYPE);
 
 CREATE TABLE WRITE_SET (
+  WS_ID bigint NOT NULL IDENTITY(1,1),
   WS_DATABASE nvarchar(128) NOT NULL,
   WS_TABLE nvarchar(256) NOT NULL,
   WS_PARTITION nvarchar(767),
   WS_TXNID bigint NOT NULL,
   WS_COMMIT_ID bigint NOT NULL,
-  WS_OPERATION_TYPE char(1) NOT NULL
+  WS_OPERATION_TYPE char(1) NOT NULL,
+  PRIMARY KEY CLUSTERED (WS_ID ASC)
 );
 
 CREATE TABLE METASTORE_DB_PROPERTIES (
@@ -1186,25 +1200,25 @@ CREATE TABLE TXN_TO_WRITE_ID (
   T2W_TXNID bigint NOT NULL,
   T2W_DATABASE nvarchar(128) NOT NULL,
   T2W_TABLE nvarchar(256) NOT NULL,
-  T2W_WRITEID bigint NOT NULL
+  T2W_WRITEID bigint NOT NULL,
+  PRIMARY KEY CLUSTERED (T2W_DATABASE ASC, T2W_TABLE ASC, T2W_TXNID ASC)
 );
 
-CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID);
 CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID);
 
 CREATE TABLE NEXT_WRITE_ID (
   NWI_DATABASE nvarchar(128) NOT NULL,
   NWI_TABLE nvarchar(256) NOT NULL,
-  NWI_NEXT bigint NOT NULL
+  NWI_NEXT bigint NOT NULL,
+  PRIMARY KEY CLUSTERED (NWI_DATABASE ASC, NWI_TABLE ASC)
 );
 
-CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE);
-
 CREATE TABLE MIN_HISTORY_WRITE_ID (
   MH_TXNID bigint NOT NULL,
   MH_DATABASE nvarchar(128) NOT NULL,
   MH_TABLE nvarchar(256) NOT NULL,
-  MH_WRITEID bigint NOT NULL
+  MH_WRITEID bigint NOT NULL,
+  PRIMARY KEY CLUSTERED (MH_TXNID ASC, MH_DATABASE ASC, MH_TABLE ASC, MH_WRITEID ASC)
 );
 
 CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);

standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.2.0-to-4.3.0.mssql.sql

@@ -5,6 +5,35 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD MRL_CAT_NAME nvarchar(128) NOT NUL
 
 CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);
 
+ALTER TABLE TXN_COMPONENTS ADD TC_ID bigint IDENTITY(1,1) NOT NULL;
+ALTER TABLE TXN_COMPONENTS ADD CONSTRAINT TXN_COMPONENTS_PK PRIMARY KEY CLUSTERED (TC_ID);
+
+ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_ID bigint IDENTITY(1,1) NOT NULL;
+ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CONSTRAINT COMPLETED_TXN_COMPONENTS_PK PRIMARY KEY CLUSTERED (CTC_ID);
+
+ALTER TABLE COMPACTION_METRICS_CACHE ADD CMC_ID bigint IDENTITY(1,1) NOT NULL;
+ALTER TABLE COMPACTION_METRICS_CACHE ADD CONSTRAINT COMPACTION_METRICS_CACHE_PK PRIMARY KEY CLUSTERED (CMC_ID);
+
+ALTER TABLE WRITE_SET ADD WS_ID bigint IDENTITY(1,1) NOT NULL;
+ALTER TABLE WRITE_SET ADD CONSTRAINT WRITE_SET_PK PRIMARY KEY CLUSTERED (WS_ID);
+
+DROP INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID;
+ALTER TABLE TXN_TO_WRITE_ID ADD CONSTRAINT TXN_TO_WRITE_ID_PK PRIMARY KEY CLUSTERED (T2W_DATABASE, T2W_TABLE, T2W_TXNID);
+
+DROP INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID;
+ALTER TABLE NEXT_WRITE_ID ADD CONSTRAINT NEXT_WRITE_ID_PK PRIMARY KEY CLUSTERED (NWI_DATABASE, NWI_TABLE);
+
+ALTER TABLE MIN_HISTORY_WRITE_ID ADD CONSTRAINT MIN_HISTORY_WRITE_ID_PK PRIMARY KEY CLUSTERED (MH_TXNID, MH_DATABASE, MH_TABLE, MH_WRITEID);
+
+ALTER TABLE TXN_LOCK_TBL ADD TLT_ID bigint IDENTITY(1,1) NOT NULL;
+ALTER TABLE TXN_LOCK_TBL ADD CONSTRAINT TXN_LOCK_TBL_PK PRIMARY KEY CLUSTERED (TLT_ID);
+
+ALTER TABLE NEXT_LOCK_ID ADD NL_ID bigint IDENTITY(1,1) NOT NULL;
+ALTER TABLE NEXT_LOCK_ID ADD CONSTRAINT NEXT_LOCK_ID_PK PRIMARY KEY CLUSTERED (NL_ID);
+
+ALTER TABLE NEXT_COMPACTION_QUEUE_ID ADD NCQI_ID bigint IDENTITY(1,1) NOT NULL;
+ALTER TABLE NEXT_COMPACTION_QUEUE_ID ADD CONSTRAINT NEXT_COMPACTION_QUEUE_ID_PK PRIMARY KEY CLUSTERED (NCQI_ID);
+
 -- These lines need to be last.  Insert any changes above.
 UPDATE VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1;
 SELECT 'Finished upgrading MetaStore schema from 4.2.0 to 4.3.0' AS MESSAGE;