PHOENIX-4229 - Parent-Child linking rows in System.Catalog break tena…

…nt view replication

phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java

@@ -32,6 +32,7 @@
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.TestUtil;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -122,36 +123,63 @@ public void testSystemCatalogWALEntryFilter() throws Exception {
 
     //now create WAL.Entry objects that refer to cells in those view rows in System.Catalog
 
-    Get tenantGet = getGet(catalogTable, TENANT_BYTES, TENANT_VIEW_NAME);
-    Get nonTenantGet = getGet(catalogTable, DEFAULT_TENANT_BYTES, NONTENANT_VIEW_NAME);
+    Get tenantViewGet = getTenantViewGet(catalogTable, TENANT_BYTES, TENANT_VIEW_NAME);
+    Get nonTenantViewGet = getTenantViewGet(catalogTable,
+        DEFAULT_TENANT_BYTES, NONTENANT_VIEW_NAME);
 
-    WAL.Entry nonTenantEntry = getEntry(systemCatalogTableName, nonTenantGet);
-    WAL.Entry tenantEntry = getEntry(systemCatalogTableName, tenantGet);
+    Get tenantLinkGet = getParentChildLinkGet(catalogTable, TENANT_BYTES, TENANT_VIEW_NAME);
+    Get nonTenantLinkGet = getParentChildLinkGet(catalogTable,
+        DEFAULT_TENANT_BYTES, NONTENANT_VIEW_NAME);
+
+    WAL.Entry nonTenantViewEntry = getEntry(systemCatalogTableName, nonTenantViewGet);
+    WAL.Entry tenantViewEntry = getEntry(systemCatalogTableName, tenantViewGet);
+
+    WAL.Entry nonTenantLinkEntry = getEntry(systemCatalogTableName, nonTenantLinkGet);
+    WAL.Entry tenantLinkEntry = getEntry(systemCatalogTableName, tenantLinkGet);
 
     //verify that the tenant view WAL.Entry passes the filter and the non-tenant view does not
     SystemCatalogWALEntryFilter filter = new SystemCatalogWALEntryFilter();
-    Assert.assertNull(filter.filter(nonTenantEntry));
-    WAL.Entry filteredTenantEntry = filter.filter(tenantEntry);
+    Assert.assertNull(filter.filter(nonTenantViewEntry));
+    WAL.Entry filteredTenantEntry = filter.filter(tenantViewEntry);
     Assert.assertNotNull("Tenant view was filtered when it shouldn't be!", filteredTenantEntry);
-    Assert.assertEquals(tenantEntry.getEdit().size(),
-        filter.filter(tenantEntry).getEdit().size());
+    Assert.assertEquals(tenantViewEntry.getEdit().size(),
+        filter.filter(tenantViewEntry).getEdit().size());
 
     //now check that a WAL.Entry with cells from both a tenant and a non-tenant
     //catalog row only allow the tenant cells through
     WALEdit comboEdit = new WALEdit();
-    comboEdit.getCells().addAll(nonTenantEntry.getEdit().getCells());
-    comboEdit.getCells().addAll(tenantEntry.getEdit().getCells());
+    comboEdit.getCells().addAll(nonTenantViewEntry.getEdit().getCells());
+    comboEdit.getCells().addAll(tenantViewEntry.getEdit().getCells());
     WAL.Entry comboEntry = new WAL.Entry(walKey, comboEdit);
 
-    Assert.assertEquals(tenantEntry.getEdit().size() + nonTenantEntry.getEdit().size()
+    Assert.assertEquals(tenantViewEntry.getEdit().size() + nonTenantViewEntry.getEdit().size()
         , comboEntry.getEdit().size());
-    Assert.assertEquals(tenantEntry.getEdit().size(),
+    Assert.assertEquals(tenantViewEntry.getEdit().size(),
         filter.filter(comboEntry).getEdit().size());
+
+    //now check that the parent-child links (which have the tenant_id of the view's parent,
+    // but are a part of the view's metadata) are migrated in the tenant case
+    // but not the non-tenant. The view's tenant_id is in th System.Catalog.COLUMN_NAME field
+
+    Assert.assertNull("Non-tenant parent-child link was not filtered " +
+        "when it should be!", filter.filter(nonTenantLinkEntry));
+    Assert.assertNotNull("Tenant parent-child link was filtered when it should not be!",
+        filter.filter(tenantLinkEntry));
+    Assert.assertEquals(tenantLinkEntry.getEdit().size(),
+        filter.filter(tenantLinkEntry).getEdit().size());
+    //add the parent-child link to the tenant view WAL entry,
+    //since they'll usually be together and they both need to
+    //be replicated
+
+    tenantViewEntry.getEdit().getCells().addAll(tenantLinkEntry.getEdit().getCells());
+    Assert.assertEquals(tenantViewEntry.getEdit().size(), tenantViewEntry.getEdit().size());
+
+
   }
 
-  public Get getGet(PTable catalogTable, byte[] tenantId, String viewName) {
+  public Get getTenantViewGet(PTable catalogTable, byte[] tenantBytes, String viewName) {
     byte[][] tenantKeyParts = new byte[5][];
-    tenantKeyParts[0] = tenantId;
+    tenantKeyParts[0] = tenantBytes;
     tenantKeyParts[1] = Bytes.toBytes(SCHEMA_NAME.toUpperCase());
     tenantKeyParts[2] = Bytes.toBytes(viewName.toUpperCase());
     tenantKeyParts[3] = Bytes.toBytes(VIEW_COLUMN_NAME);
@@ -163,6 +191,28 @@ public Get getGet(PTable catalogTable, byte[] tenantId, String viewName) {
     return new Get(key.copyBytes());
   }
 
+  public Get getParentChildLinkGet(PTable catalogTable, byte[] tenantBytes, String viewName) {
+    /* For parent-child link, the system.catalog key becomes
+      1. Parent tenant id
+      2. Parent Schema
+      3. Parent Table name
+      4. View tenant_id
+      5. Combined view SCHEMA.TABLENAME
+     */
+    byte[][] tenantKeyParts = new byte[5][];
+    tenantKeyParts[0] = null; //null tenant_id
+    tenantKeyParts[1] = null; //null parent schema
+    tenantKeyParts[2] = Bytes.toBytes(TestUtil.ENTITY_HISTORY_TABLE_NAME);
+    tenantKeyParts[3] = tenantBytes;
+    tenantKeyParts[4] = Bytes.toBytes(SchemaUtil.getTableName(SCHEMA_NAME.toUpperCase(), viewName.toUpperCase()));
+    ImmutableBytesWritable key = new ImmutableBytesWritable();
+    catalogTable.newKey(key, tenantKeyParts);
+    //the backing byte array of key might have extra space at the end.
+    // need to just slice "the good parts" which we do by calling copyBytes
+    return new Get(key.copyBytes());
+
+  }
+
   public WAL.Entry getEntry(TableName tableName, Get get) throws IOException {
     WAL.Entry entry = null;
     try(Connection conn = ConnectionFactory.createConnection(getUtility().getConfiguration())){
@@ -176,7 +226,8 @@ public WAL.Entry getEntry(TableName tableName, Get get) throws IOException {
           edit.add(c);
         }
       }
-      Assert.assertTrue("Didn't retrieve any cells from SYSTEM.CATALOG", edit.getCells().size() > 0);
+      Assert.assertTrue("Didn't retrieve any cells from SYSTEM.CATALOG",
+          edit.getCells().size() > 0);
       WALKey key = new WALKey(REGION, tableName, 0, 0, uuid);
       entry = new WAL.Entry(key, edit);
     }

phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java

@@ -22,7 +22,9 @@
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.replication.WALEntryFilter;
 import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.util.SchemaUtil;
 
 import java.util.List;
@@ -32,10 +34,15 @@
  * may change between the source and target clusters at different times, in particular
  * during cluster upgrades. However, tenant-owned data such as tenant-owned views need to
  * be copied. This WALEntryFilter will only allow tenant-owned rows in SYSTEM.CATALOG to
- * be replicated. Data from all other tables is automatically passed.
+ * be replicated. Data from all other tables is automatically passed. It will also copy
+ * child links in SYSTEM.CATALOG that are globally-owned but point to tenant-owned views.
+ *
  */
 public class SystemCatalogWALEntryFilter implements WALEntryFilter {
 
+  private static byte[] CHILD_TABLE_BYTES =
+      new byte[]{PTable.LinkType.CHILD_TABLE.getSerializedValue()};
+
   @Override
   public WAL.Entry filter(WAL.Entry entry) {
 
@@ -64,6 +71,35 @@ private boolean isTenantRowCell(Cell cell) {
     ImmutableBytesWritable key =
         new ImmutableBytesWritable(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
     //rows in system.catalog that aren't tenant-owned will have a leading separator byte
-    return key.get()[key.getOffset()] != QueryConstants.SEPARATOR_BYTE;
+    boolean isTenantRowCell = key.get()[key.getOffset()] != QueryConstants.SEPARATOR_BYTE;
+
+    /* In addition to the tenant view rows, there are parent-child links (see PHOENIX-2051) that
+     * provide an efficient way for a parent table or view to look up its children.
+     * These rows override SYSTEM_CATALOG.COLUMN_NAME with the child tenant_id,
+     * if any, and contain only a single Cell, LINK_TYPE, which is of PTable.LinkType.Child
+     */
+    boolean isChildLinkToTenantView = false;
+    if (!isTenantRowCell) {
+      ImmutableBytesWritable columnQualifier = new ImmutableBytesWritable(cell.getQualifierArray(),
+          cell.getQualifierOffset(), cell.getQualifierLength());
+      boolean isChildLink = columnQualifier.compareTo(PhoenixDatabaseMetaData.LINK_TYPE_BYTES) == 0;
+      if (isChildLink) {
+        ImmutableBytesWritable columnValue = new ImmutableBytesWritable(cell.getValueArray(),
+            cell.getValueOffset(), cell.getValueLength());
+        if (columnValue.compareTo(CHILD_TABLE_BYTES) == 0) {
+          byte[][] rowViewKeyMetadata = new byte[5][];
+          SchemaUtil.getVarChars(key.get(), key.getOffset(),
+              key.getLength(), 0, rowViewKeyMetadata);
+          //if the child link is to a tenant-owned view,
+          // the COLUMN_NAME field will be the byte[] of the tenant
+          //otherwise, it will be an empty byte array
+          // (NOT QueryConstants.SEPARATOR_BYTE, but a byte[0])
+          isChildLinkToTenantView =
+              rowViewKeyMetadata[PhoenixDatabaseMetaData.COLUMN_NAME_INDEX].length != 0;
+        }
+      }
+
+    }
+    return isTenantRowCell || isChildLinkToTenantView;
   }
 }