PHOENIX-4764 Cleanup metadata of child views for a base table that ha…

…s been dropped

phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java

@@ -429,7 +429,7 @@ AccessTestAction dropTable(final String tableName) throws SQLException {
             @Override
             public Object run() throws Exception {
                 try (Connection conn = getConnection(); Statement stmt = conn.createStatement();) {
-                    assertFalse(stmt.execute("DROP TABLE IF EXISTS " + tableName));
+                    assertFalse(stmt.execute(String.format("DROP TABLE IF EXISTS %s CASCADE", tableName)));
                 }
                 return null;
             }
@@ -654,7 +654,7 @@ AccessTestAction dropView(final String viewName) throws SQLException {
             @Override
             public Object run() throws Exception {
                 try (Connection conn = getConnection(); Statement stmt = conn.createStatement();) {
-                    assertFalse(stmt.execute("DROP VIEW " + viewName));
+                    assertFalse(stmt.execute(String.format("DROP VIEW %s CASCADE", viewName)));
                 }
                 return null;
             }

phoenix-core/src/it/java/org/apache/phoenix/end2end/DropTableWithViewsIT.java

@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.phoenix.coprocessor.TableViewFinderResult;
+import org.apache.phoenix.coprocessor.ViewFinder;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class DropTableWithViewsIT extends SplitSystemCatalogIT {
+
+    private final boolean isMultiTenant;
+    private final boolean columnEncoded;
+    private final String TENANT_SPECIFIC_URL1 = getUrl() + ';' + TENANT_ID_ATTRIB + "=" + TENANT1;
+
+    public DropTableWithViewsIT(boolean isMultiTenant, boolean columnEncoded) {
+        this.isMultiTenant = isMultiTenant;
+        this.columnEncoded = columnEncoded;
+    }
+
+    @Parameters(name="DropTableWithViewsIT_multiTenant={0}, columnEncoded={1}") // name is used by failsafe as file name in reports
+    public static Collection<Boolean[]> data() {
+        return Arrays.asList(new Boolean[][] {
+                { false, false }, { false, true },
+                { true, false }, { true, true } });
+    }
+
+    private String generateDDL(String format) {
+        return generateDDL("", format);
+    }
+
+    private String generateDDL(String options, String format) {
+        StringBuilder optionsBuilder = new StringBuilder(options);
+        if (!columnEncoded) {
+            if (optionsBuilder.length() != 0)
+                optionsBuilder.append(",");
+            optionsBuilder.append("COLUMN_ENCODED_BYTES=0");
+        }
+        if (isMultiTenant) {
+            if (optionsBuilder.length() !=0 )
+                optionsBuilder.append(",");
+            optionsBuilder.append("MULTI_TENANT=true");
+        }
+        return String.format(format, isMultiTenant ? "TENANT_ID VARCHAR NOT NULL, " : "",
+            isMultiTenant ? "TENANT_ID, " : "", optionsBuilder.toString());
+    }
+
+    @Test
+    public void testDropTableWithChildViews() throws Exception {
+        String baseTable = SchemaUtil.getTableName(SCHEMA1, generateUniqueName());
+        try (Connection conn = DriverManager.getConnection(getUrl());
+                Connection viewConn =
+                        isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL1) : conn) {
+            String ddlFormat =
+                    "CREATE TABLE IF NOT EXISTS " + baseTable + "  ("
+                            + " %s PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR "
+                            + " CONSTRAINT NAME_PK PRIMARY KEY (%s PK2)" + " ) %s";
+            conn.createStatement().execute(generateDDL(ddlFormat));
+            conn.commit();
+            // Create a view tree (i.e., tree of views) with depth of 2 and fanout factor of 4
+            for (int  i = 0; i < 4; i++) {
+                String childView = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
+                String childViewDDL = "CREATE VIEW " + childView + " AS SELECT * FROM " + baseTable;
+                viewConn.createStatement().execute(childViewDDL);
+                for (int j = 0; j < 4; j++) {
+                    String grandChildView = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
+                    String grandChildViewDDL = "CREATE VIEW " + grandChildView + " AS SELECT * FROM " + childView;
+                    viewConn.createStatement().execute(grandChildViewDDL);
+                }
+            }
+            // Drop the base table
+            String dropTable = String.format("DROP TABLE IF EXISTS %s CASCADE", baseTable);
+            conn.createStatement().execute(dropTable);
+
+            // Wait for the tasks for dropping child views to complete. The depth of the view tree is 2, so we expect that
+            // this will be done in two task handling runs, i.e., in tree task handling interval at most in general
+            // by assuming that each non-root level will be processed in one interval. To be on the safe side, we will
+            // wait at most 10 intervals.
+            long halfTimeInterval = config.getLong(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB,
+                    QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS)/2;
+            ResultSet rs = null;
+            boolean timedOut = true;
+            Thread.sleep(3 * halfTimeInterval);
+            for (int i = 3; i < 20; i++) {
+                rs = conn.createStatement().executeQuery("SELECT * " +
+                                " FROM " + PhoenixDatabaseMetaData.SYSTEM_TASK_NAME +
+                                " WHERE " + PhoenixDatabaseMetaData.TASK_TYPE + " = " +
+                                PTable.TaskType.DROP_CHILD_VIEWS.getSerializedValue());
+                Thread.sleep(halfTimeInterval);
+                if (!rs.next()) {
+                    timedOut = false;
+                    break;
+                }
+            }
+            if (timedOut) {
+                fail("Drop child view task execution timed out!");
+            }
+            // Views should be dropped by now
+            TableName linkTable = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES);
+            TableViewFinderResult childViewsResult = new TableViewFinderResult();
+            ViewFinder.findAllRelatives(getUtility().getConnection().getTable(linkTable),
+                    HConstants.EMPTY_BYTE_ARRAY,
+                    SchemaUtil.getSchemaNameFromFullName(baseTable).getBytes(),
+                    SchemaUtil.getTableNameFromFullName(baseTable).getBytes(),
+                    PTable.LinkType.CHILD_TABLE,
+                    childViewsResult);
+            assertTrue(childViewsResult.getLinks().size() == 0);
+            // There should not be any orphan views
+            rs = conn.createStatement().executeQuery("SELECT * FROM " + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME +
+                    " WHERE " + PhoenixDatabaseMetaData.TABLE_SCHEM + " = '" + SCHEMA2 +"'");
+            assertFalse(rs.next());
+        }
+    }
+}

phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java

@@ -189,6 +189,10 @@ public void testTableMetadataScan() throws SQLException {
             assertEquals(PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, rs.getString("TABLE_NAME"));
             assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
             assertTrue(rs.next());
+            assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM"));
+            assertEquals(PhoenixDatabaseMetaData.SYSTEM_TASK_TABLE, rs.getString("TABLE_NAME"));
+            assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
+            assertTrue(rs.next());
             assertEquals(null, rs.getString("TABLE_SCHEM"));
             assertEquals(tableAName, rs.getString("TABLE_NAME"));
             assertEquals(PTableType.TABLE.toString(), rs.getString("TABLE_TYPE"));

phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java

@@ -503,8 +503,10 @@ public void testTableMetadataScan() throws Exception {
             assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.TYPE_SEQUENCE, PTableType.SYSTEM);
             assertTrue(rs.next());
             assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, PTableType.SYSTEM);
+            assertTrue(rs.next());
+            assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_TASK_TABLE, PTableType.SYSTEM);
             assertFalse(rs.next());
-            
+
             rs = meta.getTables(null, "", StringUtil.escapeLike(tenantTable2), new String[] {TABLE.getValue().getString()});
             assertFalse(rs.next());
 

phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java

@@ -86,6 +86,7 @@
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -1940,7 +1941,7 @@ public void createTable(RpcController controller, CreateTableRequest request,
                 table = loadTable(env, tableKey, cacheKey, clientTimeStamp, HConstants.LATEST_TIMESTAMP,
                         clientVersion);
             } catch (ParentTableNotFoundException e) {
-                dropChildMetadata(e.getParentSchemaName(), e.getParentTableName(), e.getParentTenantId());
+                dropChildViews(env, e.getParentTenantId(), e.getParentSchemaName(), e.getParentTableName());
             }
             if (table != null) {
                 if (table.getTimeStamp() < clientTimeStamp) {
@@ -1965,7 +1966,7 @@ public void createTable(RpcController controller, CreateTableRequest request,
             // check if the table was dropped, but had child views that were have not yet
             // been cleaned up by compaction
             if (!Bytes.toString(schemaName).equals(QueryConstants.SYSTEM_SCHEMA_NAME)) {
-                dropChildMetadata(schemaName, tableName, tenantIdBytes);
+                dropChildViews(env, tenantIdBytes, schemaName, tableName);
             }
 
             byte[] parentTableKey = null;
@@ -2346,19 +2347,31 @@ public void createTable(RpcController controller, CreateTableRequest request,
         }
     }
 
-    private void dropChildMetadata(byte[] schemaName, byte[] tableName, byte[] tenantIdBytes)
+    public static void dropChildViews(RegionCoprocessorEnvironment env, byte[] tenantIdBytes, byte[] schemaName, byte[] tableName)
             throws IOException, SQLException, ClassNotFoundException {
-        TableViewFinderResult childViewsResult = new TableViewFinderResult();
-        findAllChildViews(tenantIdBytes, schemaName, tableName, childViewsResult);
+        Table hTable =
+                ServerUtil.getHTableForCoprocessorScan(env,
+                        SchemaUtil.getPhysicalTableName(
+                                PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES,
+                                env.getConfiguration()));
+        TableViewFinderResult childViewsResult = ViewFinder.findRelatedViews(hTable, tenantIdBytes, schemaName, tableName,
+                PTable.LinkType.CHILD_TABLE, HConstants.LATEST_TIMESTAMP);
+
         if (childViewsResult.hasLinks()) {
+
             for (TableInfo viewInfo : childViewsResult.getLinks()) {
                 byte[] viewTenantId = viewInfo.getTenantId();
                 byte[] viewSchemaName = viewInfo.getSchemaName();
                 byte[] viewName = viewInfo.getTableName();
+                if (logger.isDebugEnabled()) {
+                    logger.debug("dropChildViews :" + Bytes.toString(schemaName) + "." + Bytes.toString(tableName) +
+                            " -> " + Bytes.toString(viewSchemaName) + "." + Bytes.toString(viewName) +
+                            "with tenant id :" + Bytes.toString(viewTenantId));
+                }
                 Properties props = new Properties();
                 if (viewTenantId != null && viewTenantId.length != 0)
                     props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, Bytes.toString(viewTenantId));
-                try (PhoenixConnection connection = QueryUtil.getConnectionOnServer(env.getConfiguration())
+                try (PhoenixConnection connection = QueryUtil.getConnectionOnServer(props, env.getConfiguration())
                         .unwrap(PhoenixConnection.class)) {
                     MetaDataClient client = new MetaDataClient(connection);
                     org.apache.phoenix.parse.TableName viewTableName = org.apache.phoenix.parse.TableName
@@ -2369,7 +2382,6 @@ private void dropChildMetadata(byte[] schemaName, byte[] tableName, byte[] tenan
             }
         }
     }
-
     private boolean execeededIndexQuota(PTableType tableType, PTable parentTable) {
         return PTableType.INDEX == tableType && parentTable.getIndexes().size() >= maxIndexesPerTable;
     }
@@ -2533,6 +2545,7 @@ public void dropTable(RpcController controller, DropTableRequest request,
                     }
                     throw new IllegalStateException(msg);
                 }
+
                 // drop rows from catalog on this region
                 mutateRowsWithLocks(region, localMutations, Collections.<byte[]> emptySet(), HConstants.NO_NONCE,
                         HConstants.NO_NONCE);
@@ -2645,7 +2658,7 @@ private MetaDataMutationResult doDropTable(byte[] key, byte[] tenantId, byte[] s
                         EnvironmentEdgeManager.currentTimeMillis(), null);
             }
 
-            if (tableType == PTableType.TABLE || tableType == PTableType.SYSTEM) {
+            if (tableType == PTableType.TABLE || tableType == PTableType.VIEW || tableType == PTableType.SYSTEM) {
                 // check to see if the table has any child views
                 try (Table hTable =
                         ServerUtil.getHTableForCoprocessorScan(env,
@@ -2655,10 +2668,19 @@ private MetaDataMutationResult doDropTable(byte[] key, byte[] tenantId, byte[] s
                     boolean hasChildViews =
                             ViewFinder.hasChildViews(hTable, tenantId, schemaName, tableName,
                                 clientTimeStamp);
-                    if (hasChildViews && !isCascade) {
-                        // DROP without CASCADE on tables with child views is not permitted
-                        return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
-                                EnvironmentEdgeManager.currentTimeMillis(), null);
+                    if (hasChildViews) {
+                        if (!isCascade) {
+                            // DROP without CASCADE on tables with child views is not permitted
+                            return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
+                                    EnvironmentEdgeManager.currentTimeMillis(), null);
+                        }
+                        try {
+                            PhoenixConnection conn = QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class);
+                            TaskRegionObserver.addTask(conn, PTable.TaskType.DROP_CHILD_VIEWS, Bytes.toString(tenantId),
+                                Bytes.toString(schemaName), Bytes.toString(tableName), this.accessCheckEnabled);
+                        } catch (Throwable t) {
+                            logger.error("Adding a task to drop child views failed!", t);
+                        }
                     }
                 }
             }