PHOENIX-4009 Run UPDATE STATISTICS command by using MR integration on…

… snapshots
apache · Jan 17, 2019 · e3280f6 · e3280f6
1 parent 8273cc1
commit e3280f6
Show file tree

Hide file tree

Showing 30 changed files with 919 additions and 584 deletions.
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
@@ -397,6 +397,14 @@
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-common</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-metrics-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-metrics</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-common</artifactId>

diff --git a/...e/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableNonTxStatsCollectorIT.java b/...e/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableNonTxStatsCollectorIT.java
diff --git a/...core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableTxStatsCollectorIT.java b/...core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableTxStatsCollectorIT.java
diff --git a/...x-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableTxStatsCollectorIT.java b/...x-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableTxStatsCollectorIT.java
diff --git a/...nEncodedMutableNonTxStatsCollectorIT.java → ...nd/NamespaceDisabledStatsCollectorIT.java b/...nEncodedMutableNonTxStatsCollectorIT.java → ...nd/NamespaceDisabledStatsCollectorIT.java
@@ -15,24 +15,31 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.phoenix.end2end;
 
+import org.apache.phoenix.schema.stats.BaseStatsCollectorIT;
+import org.junit.runners.Parameterized;
+
 import java.util.Arrays;
 import java.util.Collection;
 
-import org.apache.phoenix.schema.stats.StatsCollectorIT;
-import org.junit.runners.Parameterized.Parameters;
+public class NamespaceDisabledStatsCollectorIT extends BaseStatsCollectorIT {
 
-public class ColumnEncodedMutableNonTxStatsCollectorIT extends StatsCollectorIT {
-
-    public ColumnEncodedMutableNonTxStatsCollectorIT(boolean mutable, String transactionProvider,
-            boolean userTableNamespaceMapped, boolean columnEncoded) {
-        super(mutable, transactionProvider, userTableNamespaceMapped, columnEncoded);
+    public NamespaceDisabledStatsCollectorIT(boolean userTableNamespaceMapped, boolean collectStatsOnSnapshot) {
+        super(userTableNamespaceMapped, collectStatsOnSnapshot);
     }
 
-    @Parameters(name = "mutable={0},transactionProvider={1},isUserTableNamespaceMapped={2},columnEncoded={3}")
-    public static Collection<Object[]> data() {
+    @Parameterized.Parameters(name = "userTableNamespaceMapped={0},collectStatsOnSnapshot={1}")
+    public static Collection<Object[]> provideData() {
         return Arrays.asList(
-            new Object[][] { { true, null, false, true }, { true, null, true, true } });
+                new Object[][] {
+                        // Collect stats on snapshots using UpdateStatisticsTool
+                        { false, true },
+                        // Collect stats via `UPDATE STATISTICS` SQL
+                        { false, false }
+                }
+        );
     }
+
 }
diff --git a/...TableNamespaceMappedStatsCollectorIT.java → ...end/NamespaceEnabledStatsCollectorIT.java b/...TableNamespaceMappedStatsCollectorIT.java → ...end/NamespaceEnabledStatsCollectorIT.java
@@ -17,48 +17,46 @@
  */
 package org.apache.phoenix.end2end;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Map;
-
+import com.google.common.collect.Maps;
 import org.apache.phoenix.query.QueryServices;
-import org.apache.phoenix.schema.stats.StatsCollectorIT;
+import org.apache.phoenix.schema.stats.BaseStatsCollectorIT;
 import org.apache.phoenix.util.ReadOnlyProps;
-import org.apache.phoenix.util.TestUtil;
 import org.junit.BeforeClass;
-import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Parameterized;
 
-import com.google.common.collect.Maps;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
 
-public class SysTableNamespaceMappedStatsCollectorIT extends StatsCollectorIT {
+public class NamespaceEnabledStatsCollectorIT extends BaseStatsCollectorIT {
 
-    public SysTableNamespaceMappedStatsCollectorIT(boolean mutable, String transactionProvider,
-            boolean userTableNamespaceMapped, boolean columnEncoded) {
-        super(mutable, transactionProvider, userTableNamespaceMapped, columnEncoded);
+    public NamespaceEnabledStatsCollectorIT(boolean userTableNamespaceMapped, boolean collectStatsOnSnapshot) {
+        super(userTableNamespaceMapped, collectStatsOnSnapshot);
     }
 
-    @Parameters(name = "mutable={0},transactionProvider={1},isUserTableNamespaceMapped={2},columnEncoded={3}")
-    public static Collection<Object[]> data() {
-        return TestUtil.filterTxParamData(Arrays.asList(
-            new Object[][] { 
-                { true, "TEPHRA", false, false }, { true, "TEPHRA", false, true }, 
-                { true, "OMID", false, false },
-            }), 1);
+    @Parameterized.Parameters(name = "userTableNamespaceMapped={0},collectStatsOnSnapshot={1}")
+    public static Collection<Object[]> provideData() {
+        return Arrays.asList(
+                new Object[][] {
+                        // Collect stats on snapshots using UpdateStatisticsTool
+                        { true, true },
+                        // Collect stats via `UPDATE STATISTICS` SQL
+                        { true, false }
+                }
+        );
     }
 
     @BeforeClass
     public static void doSetup() throws Exception {
         // enable name space mapping at global level on both client and server side
         Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(7);
-        serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+        serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.TRUE.toString());
         serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
-        serverProps.put(QueryServices.IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE, "true");
         Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
-        clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+        clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.TRUE.toString());
         clientProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
-        clientProps.put(QueryServices.IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE, "true");
         setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),
-            new ReadOnlyProps(clientProps.entrySet().iterator()));
+                new ReadOnlyProps(clientProps.entrySet().iterator()));
     }
 
 }
diff --git a/...e/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableTxStatsCollectorIT.java b/...e/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableTxStatsCollectorIT.java
diff --git a/...ncodedImmutableNonTxStatsCollectorIT.java → ...hoenix/end2end/NonTxStatsCollectorIT.java b/...ncodedImmutableNonTxStatsCollectorIT.java → ...hoenix/end2end/NonTxStatsCollectorIT.java
@@ -20,19 +20,28 @@
 import java.util.Arrays;
 import java.util.Collection;
 
-import org.apache.phoenix.schema.stats.StatsCollectorIT;
+import org.apache.phoenix.schema.stats.BaseStatsCollectorIT;
 import org.junit.runners.Parameterized.Parameters;
 
-public class NonColumnEncodedImmutableNonTxStatsCollectorIT extends StatsCollectorIT {
+public class NonTxStatsCollectorIT extends BaseStatsCollectorIT {
 
-    public NonColumnEncodedImmutableNonTxStatsCollectorIT(boolean mutable, String transactionProvider,
-            boolean userTableNamespaceMapped, boolean columnEncoded) {
-        super(mutable, transactionProvider, userTableNamespaceMapped, columnEncoded);
+    public NonTxStatsCollectorIT(boolean mutable,
+                                 String transactionProvider, boolean columnEncoded) {
+        super(mutable, transactionProvider, columnEncoded);
     }
 
-    @Parameters(name = "mutable={0},transactionProvider={1},isUserTableNamespaceMapped={2},columnEncoded={3}")
-    public static Collection<Object[]> data() {
+    @Parameters(name = "mutable={0},transactionProvider={1},columnEncoded={2}")
+    public static Collection<Object[]> provideData() {
         return Arrays.asList(
-            new Object[][] { { false, null, false, false }, { false, null, true, false } });
+                new Object[][] {
+                    // Immutable, Column Encoded
+                    { false, null, true },
+                    // Mutable, Column Encoded
+                    { true, null, true },
+                    // Immutable, Not Column Encoded
+                    { false, null, false }
+                }
+        );
     }
+
 }
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TxStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TxStatsCollectorIT.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.phoenix.schema.stats.BaseStatsCollectorIT;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+public class TxStatsCollectorIT extends BaseStatsCollectorIT {
+
+    public TxStatsCollectorIT(boolean mutable, String transactionProvider, boolean columnEncoded) {
+        super(mutable, transactionProvider, columnEncoded);
+    }
+
+    @Parameterized.Parameters(name = "mutable={0},transactionProvider={1},columnEncoded={2}")
+    public static Collection<Object[]> data() {
+        return TestUtil.filterTxParamData(
+                Arrays.asList(
+                        new Object[][] {
+                                // Immutable, TEPHRA, Column Encoded
+                                { false, "TEPHRA", true },
+                                // Immutable, TEPHRA, Non Column Encoded
+                                { false, "TEPHRA", false },
+                                // Immutable, OMID, Non Column Encoded
+                                { false, "OMID", false },
+
+                                // Mutable, TEPHRA, Column Encoded
+                                { true, "TEPHRA", true },
+                                // Mutable, TEPHRA, Non Column Encoded
+                                { true, "TEPHRA", false },
+                                // Mutable, OMID, Non Column Encoded
+                                { true, "OMID", false }}), 1);
+    }
+}