Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HBASE-22895 Fix the flakey TestSpaceQuotas #526

Merged
merged 4 commits into from
Aug 24, 2019
Merged

HBASE-22895 Fix the flakey TestSpaceQuotas #526

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f1582aa
HBASE-22895 Fix the flakey TestSpaceQuotas
jatsakthi Aug 23, 2019
679e6b5
Fix checkstyle & licensing issues
jatsakthi Aug 23, 2019
3f06cfc
Add comments to the utility functions added in the SpaceQuotaHelperFo…
jatsakthi Aug 23, 2019
b6d35aa
Put TSQBulkLoad in medium tests, TSQNonExistingTables in small tests …
jatsakthi Aug 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 174 additions & 3 deletions hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/
package org.apache.hadoop.hbase.quotas;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.IOException;
Expand All @@ -36,11 +38,19 @@
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
Expand All @@ -49,6 +59,7 @@
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.junit.rules.TestName;
import org.slf4j.Logger;
Expand All @@ -71,6 +82,7 @@ public class SpaceQuotaHelperForTests {
private final HBaseTestingUtility testUtil;
private final TestName testName;
private final AtomicLong counter;
private static final int NUM_RETRIES = 10;

public SpaceQuotaHelperForTests(
HBaseTestingUtility testUtil, TestName testName, AtomicLong counter) {
Expand Down Expand Up @@ -118,10 +130,169 @@ long listNumDefinedQuotas(Connection conn) throws IOException {
}
}

/**
* Writes the given mutation into a table until it violates the given policy.
* Verifies that the policy has been violated & then returns the name of
* the table created & written into.
*/
TableName writeUntilViolationAndVerifyViolation(
SpaceViolationPolicy policyToViolate, Mutation m) throws Exception {
final TableName tn = writeUntilViolation(policyToViolate);
verifyViolation(policyToViolate, tn, m);
return tn;
}

/**
* Writes the given mutation into a table until it violates the given policy.
* Returns the name of the table created & written into.
*/
TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Exception {
TableName tn = createTableWithRegions(10);
setQuotaLimit(tn, policyToViolate, 2L);
// Write more data than should be allowed and flush it to disk
writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);

// This should be sufficient time for the chores to run and see the change.
Thread.sleep(5000);

return tn;
}

/**
* Verifies that the given policy on the given table has been violated
*/
void verifyViolation(SpaceViolationPolicy policyToViolate, TableName tn, Mutation m)
throws Exception {
// But let's try a few times to get the exception before failing
boolean sawError = false;
String msg = "";
for (int i = 0; i < NUM_RETRIES && !sawError; i++) {
try (Table table = testUtil.getConnection().getTable(tn)) {
if (m instanceof Put) {
table.put((Put) m);
} else if (m instanceof Delete) {
table.delete((Delete) m);
} else if (m instanceof Append) {
table.append((Append) m);
} else if (m instanceof Increment) {
table.increment((Increment) m);
} else {
fail(
"Failed to apply " + m.getClass().getSimpleName() +
" to the table. Programming error");
}
LOG.info("Did not reject the " + m.getClass().getSimpleName() + ", will sleep and retry");
Thread.sleep(2000);
} catch (Exception e) {
msg = StringUtils.stringifyException(e);
if ((policyToViolate.equals(SpaceViolationPolicy.DISABLE)
&& e instanceof TableNotEnabledException) || msg.contains(policyToViolate.name())) {
LOG.info("Got the expected exception={}", msg);
sawError = true;
break;
} else {
LOG.warn("Did not get the expected exception, will sleep and retry", e);
Thread.sleep(2000);
}
}
}
if (!sawError) {
try (Table quotaTable = testUtil.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
ResultScanner scanner = quotaTable.getScanner(new Scan());
Result result = null;
LOG.info("Dumping contents of hbase:quota table");
while ((result = scanner.next()) != null) {
LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString());
}
scanner.close();
}
} else {
if (policyToViolate.equals(SpaceViolationPolicy.DISABLE)) {
assertTrue(
msg.contains("TableNotEnabledException") || msg.contains(policyToViolate.name()));
} else {
assertTrue("Expected exception message to contain the word '" + policyToViolate.name()
+ "', but was " + msg,
msg.contains(policyToViolate.name()));
}
}
assertTrue(
"Expected to see an exception writing data to a table exceeding its quota", sawError);
}

/**
* Verifies that no policy has been violated on the given table
*/
void verifyNoViolation(TableName tn, Mutation m) throws Exception {
// But let's try a few times to write data before failing
boolean sawSuccess = false;
for (int i = 0; i < NUM_RETRIES && !sawSuccess; i++) {
try (Table table = testUtil.getConnection().getTable(tn)) {
if (m instanceof Put) {
table.put((Put) m);
} else if (m instanceof Delete) {
table.delete((Delete) m);
} else if (m instanceof Append) {
table.append((Append) m);
} else if (m instanceof Increment) {
table.increment((Increment) m);
} else {
fail("Failed to apply " + m.getClass().getSimpleName() + " to the table."
+ " Programming error");
}
sawSuccess = true;
} catch (Exception e) {
LOG.info("Rejected the " + m.getClass().getSimpleName() + ", will sleep and retry");
Thread.sleep(2000);
}
}
if (!sawSuccess) {
try (Table quotaTable = testUtil.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
ResultScanner scanner = quotaTable.getScanner(new Scan());
Result result = null;
LOG.info("Dumping contents of hbase:quota table");
while ((result = scanner.next()) != null) {
LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString());
}
scanner.close();
}
}
assertTrue("Expected to succeed in writing data to a table not having quota ", sawSuccess);
}

/**
* Sets the given quota (policy & limit) on the passed table.
*/
void setQuotaLimit(final TableName tn, SpaceViolationPolicy policy, long sizeInMBs)
throws Exception {
final long sizeLimit = sizeInMBs * SpaceQuotaHelperForTests.ONE_MEGABYTE;
QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, policy);
testUtil.getAdmin().setQuota(settings);
LOG.debug("Quota limit set for table = {}, limit = {}", tn, sizeLimit);
}

/**
* Removes the space quota from the given table
*/
void removeQuotaFromtable(final TableName tn) throws Exception {
QuotaSettings removeQuota = QuotaSettingsFactory.removeTableSpaceLimit(tn);
testUtil.getAdmin().setQuota(removeQuota);
LOG.debug("Space quota settings removed from the table ", tn);
}

/**
* Removes all quotas defined in the HBase quota table.
*/
void removeAllQuotas() throws Exception {
final Connection conn = testUtil.getConnection();
removeAllQuotas(conn);
assertEquals(0, listNumDefinedQuotas(conn));
}

/**
* Removes all quotas defined in the HBase quota table.
*/
void removeAllQuotas(Connection conn) throws IOException, InterruptedException {
void removeAllQuotas(Connection conn) throws IOException {
// Wait for the quota table to be created
if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
waitForQuotaTable(conn);
Expand Down Expand Up @@ -167,14 +338,14 @@ QuotaSettings getTableSpaceQuota(Connection conn, TableName tn) throws IOExcepti
/**
* Waits 30seconds for the HBase quota table to exist.
*/
public void waitForQuotaTable(Connection conn) throws IOException {
void waitForQuotaTable(Connection conn) throws IOException {
waitForQuotaTable(conn, 30_000);
}

/**
* Waits {@code timeout} milliseconds for the HBase quota table to exist.
*/
public void waitForQuotaTable(Connection conn, long timeout) throws IOException {
void waitForQuotaTable(Connection conn, long timeout) throws IOException {
testUtil.waitFor(timeout, 1000, new Predicate<IOException>() {
@Override
public boolean evaluate() throws IOException {
Expand Down
Loading