From 2351a92cb59a84056b1f1fa7f3a6233f4f47a119 Mon Sep 17 00:00:00 2001 From: maoling Date: Wed, 8 May 2019 14:26:39 +0800 Subject: [PATCH] ZOOKEEPER-3301: Enfore the quota limit --- .../resources/markdown/zookeeperQuotas.md | 3 + .../org/apache/zookeeper/KeeperException.java | 21 +- .../java/org/apache/zookeeper/Quotas.java | 22 +- .../java/org/apache/zookeeper/StatsTrack.java | 147 +++++++++++-- .../org/apache/zookeeper/ZooKeeperMain.java | 17 +- .../org/apache/zookeeper/cli/CliCommand.java | 3 + .../zookeeper/cli/CliWrapperException.java | 2 + .../apache/zookeeper/cli/DelQuotaCommand.java | 93 +++++--- .../zookeeper/cli/ListQuotaCommand.java | 6 +- .../apache/zookeeper/cli/SetQuotaCommand.java | 204 +++++++++++++----- .../zookeeper/server/ConnectionBean.java | 2 +- .../org/apache/zookeeper/server/DataTree.java | 148 ++++++++++--- .../apache/zookeeper/server/ServerStats.java | 2 +- .../apache/zookeeper/server/ZKDatabase.java | 3 + .../org/apache/zookeeper/ZooKeeperTest.java | 4 +- .../apache/zookeeper/server/DataTreeTest.java | 10 +- .../server/DeserializationPerfTest.java | 18 +- .../server/SerializationPerfTest.java | 20 +- .../quorum/FuzzySnapshotRelatedTest.java | 3 +- .../org/apache/zookeeper/test/QuotasTest.java | 46 ++++ .../apache/zookeeper/test/StatsTrackTest.java | 131 +++++++++++ .../zookeeper/test/ZooKeeperQuotaTest.java | 144 ++++++++++++- 22 files changed, 851 insertions(+), 198 deletions(-) create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/test/QuotasTest.java create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/test/StatsTrackTest.java diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperQuotas.md b/zookeeper-docs/src/main/resources/markdown/zookeeperQuotas.md index 7f56a34cda2..7e851b653d3 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperQuotas.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperQuotas.md @@ -46,6 +46,9 @@ and `-b` (for bytes). The ZooKeeper quota is stored in ZooKeeper itself in /zookeeper/quota. To disable other people from changing the quota's set the ACL for /zookeeper/quota such that only admins are able to read and write to it. +- The quota supports the soft and hard quota.the soft quota just logs the warning info when exceeding the quota, but the hard quota +also throws a QuotaExceededException. When setting soft and hard quota on the same path, the hard quota has the priority. + ### Listing Quotas diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/KeeperException.java b/zookeeper-server/src/main/java/org/apache/zookeeper/KeeperException.java index f797bb09976..dbfb216f9ce 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/KeeperException.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/KeeperException.java @@ -146,6 +146,8 @@ public static KeeperException create(Code code) { return new ReconfigDisabledException(); case REQUESTTIMEOUT: return new RequestTimeoutException(); + case QuotaExceeded: + return new QuotaExceededException(); case OK: default: throw new IllegalArgumentException("Invalid exception code"); @@ -397,7 +399,9 @@ public static enum Code implements CodeDeprecated { /** Request not completed within max allowed time.*/ REQUESTTIMEOUT (-122), /** Attempts to perform a reconfiguration operation when reconfiguration feature is disabled. */ - RECONFIGDISABLED(-123); + RECONFIGDISABLED(-123), + /** Exceeded the quota that setted on the path.*/ + QuotaExceeded(-125); private static final Map lookup = new HashMap(); @@ -484,6 +488,8 @@ static String getCodeMessage(Code code) { return "No such watcher"; case RECONFIGDISABLED: return "Reconfig is disabled"; + case QuotaExceeded: + return "Quota has exceeded"; default: return "Unknown error " + code; } @@ -856,4 +862,17 @@ public RequestTimeoutException() { super(Code.REQUESTTIMEOUT); } } + + /** + * @see Code#QuotaExceeded + */ + @InterfaceAudience.Public + public static class QuotaExceededException extends KeeperException { + public QuotaExceededException() { + super(Code.QuotaExceeded); + } + public QuotaExceededException(String path) { + super(Code.QuotaExceeded, path); + } + } } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/Quotas.java b/zookeeper-server/src/main/java/org/apache/zookeeper/Quotas.java index b82e3395dc9..db3fff04fd4 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/Quotas.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/Quotas.java @@ -48,9 +48,19 @@ public class Quotas { * return the quota path associated with this * prefix * @param path the actual path in zookeeper. - * @return the limit quota path + * @return the quota path */ public static String quotaPath(String path) { + return quotaZookeeper + path; + } + + /** + * return the limit quota path associated with this + * prefix + * @param path the actual path in zookeeper. + * @return the limit quota path + */ + public static String limitPath(String path) { return quotaZookeeper + path + "/" + limitNode; } @@ -65,4 +75,14 @@ public static String statPath(String path) { return quotaZookeeper + path + "/" + statNode; } + + /** + * return the real path associated with this + * quotaPath. + * @param quotaPath the quotaPath which's started with /zookeeper/quota + * @return the real path associated with this quotaPath. + */ + public static String trimQuotaPath(String quotaPath) { + return quotaPath.substring(quotaZookeeper.length()); + } } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/StatsTrack.java b/zookeeper-server/src/main/java/org/apache/zookeeper/StatsTrack.java index 623c6d4464b..e987db961f5 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/StatsTrack.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/StatsTrack.java @@ -18,14 +18,25 @@ package org.apache.zookeeper; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; + /** * a class that represents the stats associated with quotas */ public class StatsTrack { - private int count; - private long bytes; - private String countStr = "count"; - private String byteStr = "bytes"; + + private static final String countStr = "count"; + private static final String countHardLimitStr = "countHardLimit"; + + private static final String byteStr = "bytes"; + private static final String byteHardLimitStr = "byteHardLimit"; + + private final Map stats = new HashMap<>(); /** * a default constructor for @@ -34,22 +45,23 @@ public class StatsTrack { public StatsTrack() { this(null); } + /** - * the stat string should be of the form count=int,bytes=long - * if stats is called with null the count and bytes are initialized - * to -1. - * @param stats the stat string to be intialized with + * the stat string should be of the form key1str=long,key2str=long,.. + * where either , or ; are valid separators + * uninitialized values are returned as -1 + * @param stats the stat string to be initialized with */ public StatsTrack(String stats) { - if (stats == null) { - stats = "count=-1,bytes=-1"; + this.stats.clear(); + if (stats == null || stats.length() == 0) { + return; } - String[] split = stats.split(","); - if (split.length != 2) { - throw new IllegalArgumentException("invalid string " + stats); + String[] keyValuePairs = stats.split("[,;]+"); + for (String keyValuePair : keyValuePairs) { + String[] kv = keyValuePair.split("="); + this.stats.put(kv[0], Long.parseLong(StringUtils.isEmpty(kv[1]) ? "-1" : kv[1])); } - count = Integer.parseInt(split[0].split("=")[1]); - bytes = Long.parseLong(split[1].split("=")[1]); } @@ -59,7 +71,7 @@ public StatsTrack(String stats) { * @return the count as part of this string */ public int getCount() { - return this.count; + return (int) getValue(countStr); } /** @@ -69,7 +81,25 @@ public int getCount() { * the count to set with */ public void setCount(int count) { - this.count = count; + setValue(countStr, count); + } + + /** + * get the count of nodes allowed as part of quota (hard limit) + * + * @return the count as part of this string + */ + public int getCountHardLimit() { + return (int) getValue(countHardLimitStr); + } + + /** + * set the count hard limit + * + * @param count the count limit to set + */ + public void setCountHardLimit(int count) { + setValue(countHardLimitStr, count); } /** @@ -78,24 +108,97 @@ public void setCount(int count) { * @return the bytes as part of this string */ public long getBytes() { - return this.bytes; + return getValue(byteStr); } /** - * set teh bytes for this stat tracker. + * set the bytes for this stat tracker. * * @param bytes * the bytes to set with */ public void setBytes(long bytes) { - this.bytes = bytes; + setValue(byteStr, bytes); + } + + /** + * get the count of bytes allowed as part of quota (hard limit) + * + * @return the bytes as part of this string + */ + public long getByteHardLimit() { + return getValue(byteHardLimitStr); + } + + /** + * set the byte hard limit + * + * @param bytes the byte limit to set + */ + public void setByteHardLimit(long bytes) { + setValue(byteHardLimitStr, bytes); + } + + /** + * get helper to lookup a given key + * + * @param key the key to lookup + * @return key's value or -1 if it doesn't exist + */ + private long getValue(String key) { + Long val = this.stats.get(key); + return val == null ? -1 : val.longValue(); + } + + /** + * set helper to set the value for the specified key + * + * @param key the key to set + * @param value the value to set + */ + private void setValue(String key, long value) { + this.stats.put(key, value); } - @Override /* * returns the string that maps to this stat tracking. + * + * Builds a string of the form + * "count=4,bytes=5=;countHardLimit=10;byteHardLimit=10" + * + * This string is slightly hacky to preserve compatibility with 3.4.3 and + * older parser. In particular, count must be first, bytes must be second, + * all new fields must use a separator that is not a "," (so, ";"), and the + * seemingly spurious "=" after the bytes field is essential to allowing + * it to be parseable by the old parsing code. */ + @Override public String toString() { - return countStr + "=" + count + "," + byteStr + "=" + bytes; + StringBuilder buf = new StringBuilder(); + ArrayList keys = new ArrayList<>(stats.keySet()); + + // Special handling for count=,byte= to enforce them coming first + // for backwards compatibility + keys.remove(countStr); + keys.remove(byteStr); + buf.append(countStr); + buf.append("="); + buf.append(getCount()); + buf.append(","); + buf.append(byteStr); + buf.append("="); + buf.append(getBytes()); + if (!keys.isEmpty()) { + // Add extra = to trick old parsing code so it will ignore new flags + buf.append("="); + Collections.sort(keys); + for(String key : keys) { + buf.append(";"); + buf.append(key); + buf.append("="); + buf.append(stats.get(key)); + } + } + return buf.toString(); } } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/ZooKeeperMain.java b/zookeeper-server/src/main/java/org/apache/zookeeper/ZooKeeperMain.java index ce575b19628..f6e47d28ab7 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/ZooKeeperMain.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/ZooKeeperMain.java @@ -295,8 +295,7 @@ protected void connectToZK(String newHost) throws InterruptedException, IOExcept zk = new ZooKeeperAdmin(host, Integer.parseInt(cl.getOption("timeout")), new MyWatcher(), readOnly); } - public static void main(String args[]) throws CliException, IOException, InterruptedException - { + public static void main(String args[]) throws CliException, IOException, InterruptedException, KeeperException { ZooKeeperMain main = new ZooKeeperMain(args); main.run(); } @@ -311,7 +310,7 @@ public ZooKeeperMain(ZooKeeper zk) { this.zk = zk; } - void run() throws CliException, IOException, InterruptedException { + void run() throws CliException, IOException, InterruptedException, KeeperException { if (cl.getCommand() == null) { System.out.println("Welcome to ZooKeeper!"); @@ -372,7 +371,7 @@ void run() throws CliException, IOException, InterruptedException { System.exit(exitCode); } - public void executeLine(String line) throws CliException, InterruptedException, IOException { + public void executeLine(String line) throws CliException, InterruptedException, IOException, KeeperException { if (!line.equals("")) { cl.parseCommand(line); addToHistory(commandCount,line); @@ -425,8 +424,8 @@ public static boolean delQuota(ZooKeeper zk, String path, boolean bytes, boolean numNodes) throws KeeperException, IOException, InterruptedException { - String parentPath = Quotas.quotaZookeeper + path; - String quotaPath = Quotas.quotaZookeeper + path + "/" + Quotas.limitNode; + String parentPath = Quotas.quotaPath(path); + String quotaPath = Quotas.limitPath(path); if (zk.exists(quotaPath, false) == null) { System.out.println("Quota does not exist for " + path); return true; @@ -518,7 +517,7 @@ public static boolean createQuota(ZooKeeper zk, String path, // are not the children then this path // is an ancestor of some path that // already has quota - String realPath = Quotas.quotaZookeeper + path; + String realPath = Quotas.quotaPath(path); try { List children = zk.getChildren(realPath, false); for (String child: children) { @@ -589,7 +588,7 @@ public static boolean createQuota(ZooKeeper zk, String path, return true; } - protected boolean processCmd(MyCommandOptions co) throws CliException, IOException, InterruptedException { + protected boolean processCmd(MyCommandOptions co) throws CliException, IOException, InterruptedException, KeeperException { boolean watch = false; try { watch = processZKCmd(co); @@ -601,7 +600,7 @@ protected boolean processCmd(MyCommandOptions co) throws CliException, IOExcepti return watch; } - protected boolean processZKCmd(MyCommandOptions co) throws CliException, IOException, InterruptedException { + protected boolean processZKCmd(MyCommandOptions co) throws CliException, IOException, InterruptedException, KeeperException { String[] args = co.getArgArray(); String cmd = co.getCommand(); if (args.length < 1) { diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/CliCommand.java b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/CliCommand.java index cd84175adbb..ef0cff6ff94 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/CliCommand.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/CliCommand.java @@ -17,8 +17,11 @@ */ package org.apache.zookeeper.cli; +import java.io.IOException; import java.io.PrintStream; import java.util.Map; + +import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooKeeper; /** diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/CliWrapperException.java b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/CliWrapperException.java index c0fafda19d3..d64729fe289 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/CliWrapperException.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/CliWrapperException.java @@ -53,6 +53,8 @@ private static String getMessage(Throwable cause) { return "No quorum of new config is connected and " + "up-to-date with the leader of last commmitted config - try invoking reconfiguration after " + "new servers are connected and synced"; + } else if (keeperException instanceof KeeperException.QuotaExceededException) { + return "Quota has exceeded : " + keeperException.getPath(); } } return cause.getMessage(); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/DelQuotaCommand.java b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/DelQuotaCommand.java index 0718d1dc2bb..f0e688d82ba 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/DelQuotaCommand.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/DelQuotaCommand.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; + import org.apache.commons.cli.*; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.Quotas; @@ -36,11 +37,14 @@ public class DelQuotaCommand extends CliCommand { private CommandLine cl; public DelQuotaCommand() { - super("delquota", "[-n|-b] path"); + super("delquota", "[-n|-b|-N|-B] path"); OptionGroup og1 = new OptionGroup(); - og1.addOption(new Option("b", false, "bytes quota")); - og1.addOption(new Option("n", false, "num quota")); + og1.addOption(new Option("n", false, "num soft quota")); + og1.addOption(new Option("b", false, "bytes soft quota")); + og1.addOption(new Option("N", false, "num hard quota")); + og1.addOption(new Option("B", false, "bytes hard quota")); + options.addOptionGroup(og1); } @@ -62,22 +66,35 @@ public CliCommand parse(String[] cmdArgs) throws CliParseException { @Override public boolean exec() throws CliException { - //if neither option -n or -b is specified, we delete - // the quota node for this node. String path = args[1]; + // Use a StatsTrack object to pass in to delQuota which quotas + // to delete by setting them to 1 as a flag. + StatsTrack quota = new StatsTrack(); + if (cl.hasOption("n")) { + quota.setCount(1); + } + if (cl.hasOption("b")) { + quota.setBytes(1); + } + if (cl.hasOption("N")) { + quota.setCountHardLimit(1); + } + if (cl.hasOption("B")) { + quota.setByteHardLimit(1); + } + + boolean flagSet = (cl.hasOption("n") || cl.hasOption("N") || + cl.hasOption("b") || cl.hasOption("B")); try { - if (cl.hasOption("b")) { - delQuota(zk, path, true, false); - } else if (cl.hasOption("n")) { - delQuota(zk, path, false, true); - } else if (args.length == 2) { - // we don't have an option specified. - // just delete whole quota node - delQuota(zk, path, true, true); - } - } catch (KeeperException|InterruptedException|IOException ex) { + delQuota(zk, path, flagSet ? quota : null); + } catch (IllegalArgumentException ex) { + throw new MalformedPathException(ex.getMessage()); + } catch (KeeperException.NoNodeException ne) { + err.println("quota for " + path + " does not exist."); + } catch (KeeperException|InterruptedException ex) { throw new CliWrapperException(ex); } + return false; } @@ -86,19 +103,16 @@ public boolean exec() throws CliException { * * @param zk the zookeeper client * @param path the path to delete quota for - * @param bytes true if number of bytes needs to be unset - * @param numNodes true if number of nodes needs to be unset + * @param quota the quotas to delete (set to 1), null to delete all * @return true if quota deletion is successful * @throws KeeperException - * @throws IOException + * @throws MalformedPathException * @throws InterruptedException */ - public static boolean delQuota(ZooKeeper zk, String path, - boolean bytes, boolean numNodes) - throws KeeperException, IOException, InterruptedException, MalformedPathException { - String parentPath = Quotas.quotaZookeeper + path; - String quotaPath = Quotas.quotaZookeeper + path + "/" + - Quotas.limitNode; + public static boolean delQuota(ZooKeeper zk, String path, StatsTrack quota) + throws KeeperException, InterruptedException, MalformedPathException { + String parentPath = Quotas.quotaPath(path); + String quotaPath = Quotas.limitPath(path); if (zk.exists(quotaPath, false) == null) { System.out.println("Quota does not exist for " + path); return true; @@ -109,17 +123,11 @@ public static boolean delQuota(ZooKeeper zk, String path, } catch (IllegalArgumentException ex) { throw new MalformedPathException(ex.getMessage()); } catch (KeeperException.NoNodeException ne) { - System.err.println("quota does not exist for " + path); - return true; + throw new KeeperException.NoNodeException(ne.getMessage()); } StatsTrack strack = new StatsTrack(new String(data)); - if (bytes && !numNodes) { - strack.setBytes(-1L); - zk.setData(quotaPath, strack.toString().getBytes(), -1); - } else if (!bytes && numNodes) { - strack.setCount(-1); - zk.setData(quotaPath, strack.toString().getBytes(), -1); - } else if (bytes && numNodes) { + + if (quota == null) { // delete till you can find a node with more than // one child List children = zk.getChildren(parentPath, false); @@ -129,7 +137,23 @@ public static boolean delQuota(ZooKeeper zk, String path, } // cut the tree till their is more than one child trimProcQuotas(zk, parentPath); + } else { + if (quota.getCount() > 0) { + strack.setCount(-1); + } + if (quota.getBytes() > 0) { + strack.setBytes(-1L); + } + if (quota.getCountHardLimit() > 0) { + strack.setCountHardLimit(-1); + } + if (quota.getByteHardLimit() > 0) { + strack.setByteHardLimit(-1L); + } + + zk.setData(quotaPath, strack.toString().getBytes(), -1); } + return true; } @@ -141,11 +165,10 @@ public static boolean delQuota(ZooKeeper zk, String path, * unwanted parent in the path. * @return true if successful * @throws KeeperException - * @throws IOException * @throws InterruptedException */ private static boolean trimProcQuotas(ZooKeeper zk, String path) - throws KeeperException, IOException, InterruptedException { + throws KeeperException, InterruptedException { if (Quotas.quotaZookeeper.equals(path)) { return true; } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/ListQuotaCommand.java b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/ListQuotaCommand.java index d2521adca37..dd59365076d 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/ListQuotaCommand.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/ListQuotaCommand.java @@ -55,8 +55,7 @@ public CliCommand parse(String[] cmdArgs) throws CliParseException { @Override public boolean exec() throws CliException { String path = args[1]; - String absolutePath = Quotas.quotaZookeeper + path + "/" - + Quotas.limitNode; + String absolutePath = Quotas.limitPath(path); try { err.println("absolute path is " + absolutePath); Stat stat = new Stat(); @@ -65,8 +64,7 @@ public boolean exec() throws CliException { out.println("Output quota for " + path + " " + st.toString()); - data = zk.getData(Quotas.quotaZookeeper + path + "/" - + Quotas.statNode, false, stat); + data = zk.getData(Quotas.statPath(path), false, stat); out.println("Output stat for " + path + " " + new StatsTrack(new String(data)).toString()); } catch (IllegalArgumentException ex) { diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/SetQuotaCommand.java b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/SetQuotaCommand.java index 81194541b90..5a7196cce81 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/SetQuotaCommand.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/SetQuotaCommand.java @@ -17,7 +17,6 @@ */ package org.apache.zookeeper.cli; -import java.io.IOException; import java.util.List; import org.apache.commons.cli.*; import org.apache.zookeeper.*; @@ -36,14 +35,17 @@ public class SetQuotaCommand extends CliCommand { private CommandLine cl; public SetQuotaCommand() { - super("setquota", "-n|-b val path"); - + super("setquota", "-n|-b|-N|-B val path"); + OptionGroup og1 = new OptionGroup(); - og1.addOption(new Option("b", true, "bytes quota")); - og1.addOption(new Option("n", true, "num quota")); + og1.addOption(new Option("n", true, "num soft quota")); + og1.addOption(new Option("b", true, "bytes soft quota")); + og1.addOption(new Option("N", true, "num hard quota")); + og1.addOption(new Option("B", true, "bytes hard quota")); + og1.setRequired(true); options.addOptionGroup(og1); - } + } @Override public CliCommand parse(String[] cmdArgs) throws CliParseException { @@ -66,34 +68,95 @@ public boolean exec() throws CliException { // get the args String path = args[1]; - if (cl.hasOption("b")) { - // we are setting the bytes quota - long bytes = Long.parseLong(cl.getOptionValue("b")); - try { - createQuota(zk, path, bytes, -1); - } catch (KeeperException|IOException|InterruptedException ex) { - throw new CliWrapperException(ex); - } - } else if (cl.hasOption("n")) { - // we are setting the num quota - int numNodes = Integer.parseInt(cl.getOptionValue("n")); + StatsTrack quota = new StatsTrack(); + quota.setCount(-1); + quota.setBytes(-1L); + quota.setCountHardLimit(-1); + quota.setByteHardLimit(-1L); + + if (!checkOptionValue(quota)) { + return false; + } + + boolean flagSet = (cl.hasOption("n") || cl.hasOption("N") || + cl.hasOption("b") || cl.hasOption("B")); + if (flagSet) { try { - createQuota(zk, path, -1L, numNodes); - } catch (KeeperException|IOException|InterruptedException ex) { + createQuota(zk, path, quota); + } catch (IllegalArgumentException ex) { + throw new MalformedPathException(ex.getMessage()); + } catch (KeeperException|InterruptedException ex) { throw new CliWrapperException(ex); } } else { - throw new MalformedCommandException(getUsageStr()); + err.println(getUsageStr()); } return false; } - public static boolean createQuota(ZooKeeper zk, String path, - long bytes, int numNodes) - throws KeeperException, IOException, InterruptedException, MalformedPathException { + private boolean checkOptionValue(StatsTrack quota) { + + try { + if (cl.hasOption("n")) { + // we are setting the num quota + int count = Integer.parseInt(cl.getOptionValue("n")); + if (count > 0) { + quota.setCount(count); + } else { + err.println("the num quota must be greater than zero"); + return false; + } + } + if (cl.hasOption("b")) { + // we are setting the bytes quota + long bytes = Long.parseLong(cl.getOptionValue("b")); + if (bytes >= 0) { + quota.setBytes(bytes); + } else { + err.println("the bytes quota must be greater than or equal to zero"); + return false; + } + } + if (cl.hasOption("N")) { + // we are setting the num hard quota + int count = Integer.parseInt(cl.getOptionValue("N")); + if (count > 0) { + quota.setCountHardLimit(count); + } else { + err.println("the num quota must be greater than zero"); + return false; + } + } + if (cl.hasOption("B")) { + // we are setting the byte hard quota + long bytes = Long.parseLong(cl.getOptionValue("B")); + if (bytes >= 0) { + quota.setByteHardLimit(bytes); + } else { + err.println("the bytes quota must be greater than or equal to zero"); + return false; + } + } + } catch (NumberFormatException e) { + err.println("NumberFormatException happens when parsing the option value"); + return false; + } + + return true; + } + + /** + * this method creates a quota node for the path + * @param zk the ZooKeeper client + * @param path the path for which quota needs to be created + * @param quota the quotas + * @return true if its successful and false if not. + */ + public static boolean createQuota(ZooKeeper zk, String path, StatsTrack quota) + throws KeeperException, InterruptedException, MalformedPathException { // check if the path exists. We cannot create - // quota for a path that already exists in zookeeper + // quota for a path that doesn't exist in zookeeper // for now. Stat initStat; try { @@ -113,7 +176,7 @@ public static boolean createQuota(ZooKeeper zk, String path, // are not the children then this path // is an ancestor of some path that // already has quota - String realPath = Quotas.quotaZookeeper + path; + String realPath = Quotas.quotaPath(path); try { List children = zk.getChildren(realPath, false); for (String child : children) { @@ -150,37 +213,74 @@ public static boolean createQuota(ZooKeeper zk, String path, for (int i = 1; i < splits.length; i++) { sb.append("/").append(splits[i]); quotaPath = sb.toString(); - try { - zk.create(quotaPath, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - } catch (KeeperException.NodeExistsException ne) { - //do nothing + if (zk.exists(quotaPath, false) == null) { + try { + zk.create(quotaPath, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } catch (KeeperException.NodeExistsException ne) { + //do nothing + } } } String statPath = quotaPath + "/" + Quotas.statNode; quotaPath = quotaPath + "/" + Quotas.limitNode; - StatsTrack strack = new StatsTrack(null); - strack.setBytes(bytes); - strack.setCount(numNodes); - try { - zk.create(quotaPath, strack.toString().getBytes(), + byte[] data; + + if (zk.exists(quotaPath, false) == null) { + zk.create(quotaPath, quota.toString().getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - StatsTrack stats = new StatsTrack(null); - stats.setBytes(0L); + + StatsTrack stats = new StatsTrack(); stats.setCount(0); + stats.setBytes(0L); + zk.create(statPath, stats.toString().getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } catch (KeeperException.NodeExistsException ne) { - byte[] data = zk.getData(quotaPath, false, new Stat()); - StatsTrack strackC = new StatsTrack(new String(data)); - if (bytes != -1L) { - strackC.setBytes(bytes); + + data = zk.getData(quotaPath, false, new Stat()); + StatsTrack quotaStrack = new StatsTrack(new String(data)); + + data = zk.getData(statPath, false, new Stat()); + StatsTrack statStrack = new StatsTrack(new String(data)); + if ((quotaStrack.getCount() > -1 && quotaStrack.getCount() < statStrack.getCount()) || (quotaStrack.getCountHardLimit() > -1 + && quotaStrack.getCountHardLimit() < statStrack.getCount())) { + System.out.println("[Warning]: the count quota you create is less than the existing count:" + statStrack.getCount()); + } + if ((quotaStrack.getBytes() > -1 && quotaStrack.getBytes() < statStrack.getBytes()) || (quotaStrack.getByteHardLimit() > -1 + && quotaStrack.getByteHardLimit() < statStrack.getBytes())) { + System.out.println("[Warning]: the bytes quota you create is less than the existing bytes:" + statStrack.getBytes()); + } + } else { + data = zk.getData(quotaPath, false, new Stat()); + StatsTrack quotaStrack = new StatsTrack(new String(data)); + + if (quota.getCount() > -1) { + quotaStrack.setCount(quota.getCount()); + } + if (quota.getBytes() > -1L) { + quotaStrack.setBytes(quota.getBytes()); } - if (numNodes != -1) { - strackC.setCount(numNodes); + if (quota.getCountHardLimit() > -1) { + quotaStrack.setCountHardLimit(quota.getCountHardLimit()); } - zk.setData(quotaPath, strackC.toString().getBytes(), -1); + if (quota.getByteHardLimit() > -1L) { + quotaStrack.setByteHardLimit(quota.getByteHardLimit()); + } + + data = zk.getData(statPath, false, new Stat()); + StatsTrack statStrack = new StatsTrack(new String(data)); + if ((quota.getCount() > -1 && quotaStrack.getCount() < statStrack.getCount()) || (quota.getCountHardLimit() > -1 + && quotaStrack.getCountHardLimit() < statStrack.getCount())) { + System.out.println("[Warning]: the count quota you set is less than the existing count:" + statStrack.getCount()); + } + if ((quota.getBytes() > -1 && quotaStrack.getBytes() < statStrack.getBytes()) || (quota.getByteHardLimit() > -1 + && quotaStrack.getByteHardLimit() < statStrack.getBytes())) { + System.out.println("[Warning]: the bytes quota you set is less than the existing bytes:" + statStrack.getBytes()); + } + + zk.setData(quotaPath, quotaStrack.toString().getBytes(), -1); } + return true; } @@ -188,13 +288,13 @@ private static void checkIfParentQuota(ZooKeeper zk, String path) throws InterruptedException, KeeperException { final String[] splits = path.split("/"); String quotaPath = Quotas.quotaZookeeper; - for (String str : splits) { - if (str.length() == 0) { - // this should only be for the beginning of the path - // i.e. "/..." - split(path)[0] is empty string before first '/' - continue; - } - quotaPath += "/" + str; + + StringBuilder sb = new StringBuilder(); + sb.append(quotaPath); + for (int i = 1; i < splits.length - 1; i++) { + sb.append("/"); + sb.append(splits[i]); + quotaPath = sb.toString(); List children = null; try { children = zk.getChildren(quotaPath, false); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ConnectionBean.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ConnectionBean.java index 58917e05f2b..0b5d3ee3f27 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ConnectionBean.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ConnectionBean.java @@ -134,7 +134,7 @@ public long getPacketsReceived() { public long getPacketsSent() { return stats.getPacketsSent(); } - + public int getSessionTimeout() { return connection.getSessionTimeout(); } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java index ce27c42ea8f..681bc96d109 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java @@ -18,6 +18,7 @@ package org.apache.zookeeper.server; +import org.apache.commons.lang.StringUtils; import org.apache.jute.InputArchive; import org.apache.jute.OutputArchive; import org.apache.jute.Record; @@ -25,6 +26,7 @@ import org.apache.zookeeper.KeeperException.Code; import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.KeeperException.QuotaExceededException; import org.apache.zookeeper.Quotas; import org.apache.zookeeper.StatsTrack; import org.apache.zookeeper.WatchedEvent; @@ -347,43 +349,104 @@ static public void copyStat(Stat from, Stat to) { * @param countDiff * the diff to be added to the count */ - public void updateCountBytes(String lastPrefix, long bytesDiff, int countDiff) { - String statNode = Quotas.statPath(lastPrefix); - DataNode node = nodes.get(statNode); + public void updateQuotaStat(String lastPrefix, long bytesDiff, int countDiff) { - StatsTrack updatedStat = null; - if (node == null) { + String statNodePath = Quotas.statPath(lastPrefix); + DataNode statNode = nodes.get(statNodePath); + + StatsTrack updatedStat; + if (statNode == null) { // should not happen - LOG.error("Missing count node for stat " + statNode); + LOG.error("Missing node for stat " + statNode); return; } - synchronized (node) { - updatedStat = new StatsTrack(new String(node.data)); + synchronized (statNode) { + updatedStat = new StatsTrack(new String(statNode.data)); updatedStat.setCount(updatedStat.getCount() + countDiff); updatedStat.setBytes(updatedStat.getBytes() + bytesDiff); - node.data = updatedStat.toString().getBytes(); + + statNode.data = updatedStat.toString().getBytes(); + } + } + + /** + * check whether exceeded the quota. + * + * @param lastPrefix + * the path of the node that is quotaed. + * @param bytesDiff + * the diff to be added to number of bytes + * @param countDiff + * the diff to be added to the count + */ + private void checkQuota(String lastPrefix, long bytesDiff, int countDiff) + throws KeeperException.QuotaExceededException { + LOG.debug("checkQuota: lastPrefix={}, bytesDiff={}, countDiff={}", lastPrefix, bytesDiff, countDiff); + + if (StringUtils.isEmpty(lastPrefix)) { + return; } // now check if the counts match the quota - String quotaNode = Quotas.quotaPath(lastPrefix); - node = nodes.get(quotaNode); - StatsTrack thisStats = null; + String limitNode = Quotas.limitPath(lastPrefix); + DataNode node = nodes.get(limitNode); + StatsTrack limitStats; + if (node == null) { + // should not happen + LOG.error("Missing count node for quota " + limitNode); + return; + } + synchronized (node) { + limitStats = new StatsTrack(new String(node.data)); + } + //check the quota + boolean checkCountQuota = countDiff > 0 && + (limitStats.getCount() > -1 || limitStats.getCountHardLimit() > -1); + boolean checkByteQuota = bytesDiff > 0 && + (limitStats.getBytes() > -1 || limitStats.getByteHardLimit() > -1); + + if (!checkCountQuota && !checkByteQuota) { + return; + } + + //check the statPath quota + String statNode = Quotas.statPath(lastPrefix); + node = nodes.get(statNode); + + StatsTrack currentStats; if (node == null) { // should not happen - LOG.error("Missing count node for quota " + quotaNode); + LOG.error("Missing count node for stat " + statNode); return; } synchronized (node) { - thisStats = new StatsTrack(new String(node.data)); + currentStats = new StatsTrack(new String(node.data)); } - if (thisStats.getCount() > -1 && (thisStats.getCount() < updatedStat.getCount())) { - LOG.warn("Quota exceeded: " + lastPrefix + " count=" - + updatedStat.getCount() + " limit=" - + thisStats.getCount()); + + //check + if (checkCountQuota) { + long newCount = currentStats.getCount() + countDiff; + boolean isCountHardLimit = limitStats.getCountHardLimit() > -1 ? true : false; + long countLimit = isCountHardLimit ? limitStats.getCountHardLimit() : limitStats.getCount(); + + if (newCount > countLimit) { + String msg = lastPrefix + " [count=" + newCount + ", " + (isCountHardLimit ? "hard" : "soft") + "limit=" + countLimit + "]"; + LOG.warn("Quota exceeded: {}", msg); + if (isCountHardLimit) { + throw new KeeperException.QuotaExceededException(lastPrefix); + } + } } - if (thisStats.getBytes() > -1 && (thisStats.getBytes() < updatedStat.getBytes())) { - LOG.warn("Quota exceeded: " + lastPrefix + " bytes=" - + updatedStat.getBytes() + " limit=" - + thisStats.getBytes()); + if (checkByteQuota) { + long newBytes = currentStats.getBytes() + bytesDiff; + boolean isByteHardLimit = limitStats.getByteHardLimit() > -1 ? true : false; + long byteLimit = isByteHardLimit ? limitStats.getByteHardLimit() : limitStats.getBytes(); + if (newBytes > byteLimit) { + String msg = lastPrefix + " [bytes=" + newBytes + ", " + (isByteHardLimit ? "hard" : "soft") + "limit=" + byteLimit + "]"; + LOG.warn("Quota exceeded: {}", msg); + if (isByteHardLimit) { + throw new KeeperException.QuotaExceededException(lastPrefix); + } + } } } @@ -406,7 +469,7 @@ public void updateCountBytes(String lastPrefix, long bytesDiff, int countDiff) { */ public void createNode(final String path, byte data[], List acl, long ephemeralOwner, int parentCVersion, long zxid, long time) - throws NoNodeException, NodeExistsException { + throws NoNodeException, NodeExistsException, QuotaExceededException { createNode(path, data, acl, ephemeralOwner, parentCVersion, zxid, time, null); } @@ -432,7 +495,16 @@ public void createNode(final String path, byte data[], List acl, public void createNode(final String path, byte data[], List acl, long ephemeralOwner, int parentCVersion, long zxid, long time, Stat outputStat) throws KeeperException.NoNodeException, - KeeperException.NodeExistsException { + KeeperException.NodeExistsException, + KeeperException.QuotaExceededException { + + //check the quota firstly + String lastPrefix = getMaxPrefixWithQuota(path); + long bytes = data == null ? 0 : data.length; + if (lastPrefix != null) { + checkQuota(lastPrefix, bytes, 1); + } + int lastSlash = path.lastIndexOf('/'); String parentName = path.substring(0, lastSlash); String childName = path.substring(lastSlash + 1); @@ -518,12 +590,11 @@ public void createNode(final String path, byte data[], List acl, } } // also check to update the quotas for this node - String lastPrefix = getMaxPrefixWithQuota(path); - long bytes = data == null ? 0 : data.length; - if(lastPrefix != null) { + if (lastPrefix != null) { // ok we have some match and need to update - updateCountBytes(lastPrefix, bytes, 1); + updateQuotaStat(lastPrefix, bytes, 1); } + updateWriteStat(path, bytes); dataWatches.triggerWatch(path, Event.EventType.NodeCreated); childWatches.triggerWatch(parentName.equals("") ? "/" : parentName, @@ -602,11 +673,11 @@ public void deleteNode(String path, long zxid) String lastPrefix = getMaxPrefixWithQuota(path); if(lastPrefix != null) { // ok we have some match and need to update - int bytes = 0; + long bytes = 0; synchronized (node) { bytes = (node.data == null ? 0 : -(node.data.length)); } - updateCountBytes(lastPrefix, bytes,-1); + updateQuotaStat(lastPrefix, bytes, -1); } updateWriteStat(path, 0L); @@ -625,7 +696,8 @@ public void deleteNode(String path, long zxid) } public Stat setData(String path, byte data[], int version, long zxid, - long time) throws KeeperException.NoNodeException { + long time) throws KeeperException.NoNodeException, + KeeperException.QuotaExceededException { Stat s = new Stat(); DataNode n = nodes.get(path); if (n == null) { @@ -634,6 +706,14 @@ public Stat setData(String path, byte data[], int version, long zxid, byte lastdata[] = null; synchronized (n) { lastdata = n.data; + } + // first do a quota check if the path is in a quota subtree. + String lastPrefix = getMaxPrefixWithQuota(path); + long bytesDiff = (data == null ? 0 : data.length) - (lastdata == null ? 0 : lastdata.length); + if (lastPrefix != null) { + checkQuota(lastPrefix, bytesDiff,0); + } + synchronized (n) { n.data = data; n.stat.setMtime(time); n.stat.setMzxid(zxid); @@ -641,11 +721,9 @@ public Stat setData(String path, byte data[], int version, long zxid, n.copyStat(s); } // now update if the path is in a quota subtree. - String lastPrefix = getMaxPrefixWithQuota(path); long dataBytes = data == null ? 0 : data.length; - if(lastPrefix != null) { - this.updateCountBytes(lastPrefix, dataBytes - - (lastdata == null ? 0 : lastdata.length), 0); + if (lastPrefix != null) { + updateQuotaStat(lastPrefix, bytesDiff, 0); } nodeDataSize.addAndGet(getNodeSize(path, data) - getNodeSize(path, lastdata)); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerStats.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerStats.java index 9e5a15e6f82..516c83aee11 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerStats.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerStats.java @@ -99,7 +99,7 @@ public long getPacketsSent() { public String getServerState() { return provider.getState(); } - + /** The number of client connections alive to this server */ public int getNumAliveClientConnections() { return provider.getNumAliveConnections(); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java index 8ad79ca7689..7a4768c41f5 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java @@ -41,6 +41,7 @@ import org.apache.jute.Record; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NoNodeException; +import org.apache.zookeeper.KeeperException.QuotaExceededException; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.Watcher.WatcherType; import org.apache.zookeeper.ZooDefs; @@ -640,6 +641,8 @@ public synchronized void initConfigInZKDatabase(QuorumVerifier qv) { this.dataTree.setData(ZooDefs.CONFIG_NODE, qv.toString().getBytes(), -1, qv.getVersion(), Time.currentWallTime()); } catch (NoNodeException e) { System.out.println("configuration node missing - should not happen"); + } catch (QuotaExceededException e) { + System.out.println("configuration node has exceeded the quota - should not happen"); } } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/ZooKeeperTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/ZooKeeperTest.java index 3c85409d54c..e7c96c6228e 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/ZooKeeperTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/ZooKeeperTest.java @@ -246,7 +246,9 @@ public void testStatWhenPathDoesNotExist() throws IOException, Assert.fail("As Node does not exist, command should fail by throwing No Node Exception."); } catch (CliException e) { Assert.assertEquals("Node does not exist: /invalidPath", e.getMessage()); - } + } catch (KeeperException e) { + e.printStackTrace(); + } } @Test diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java index 931fdac60fe..dddda410a9c 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java @@ -22,11 +22,11 @@ import org.slf4j.LoggerFactory; import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.KeeperException.QuotaExceededException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZKTestCase; import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; import org.junit.After; import org.junit.Assert; @@ -45,13 +45,11 @@ import org.apache.jute.Record; import org.apache.zookeeper.common.PathTrie; import java.lang.reflect.*; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.List; -import java.util.ArrayList; + import org.apache.zookeeper.metrics.MetricsUtils; public class DataTreeTest extends ZKTestCase { @@ -114,7 +112,7 @@ private void killZkClientSession(long session, long zxid, } private void createEphemeralNode(long session, final DataTree dataTree, - int count) throws NoNodeException, NodeExistsException { + int count) throws NoNodeException, NodeExistsException, QuotaExceededException { for (int i = 0; i < count; i++) { dataTree.createNode("/test" + i, new byte[0], null, session + i, dataTree.getNode("/").stat.getCversion() + 1, 1, 1); @@ -205,7 +203,7 @@ public void testPathTrieClearOnDeserialize() throws Exception { dserTree.createNode("/bug", new byte[20], null, -1, 1, 1, 1); dserTree.createNode(Quotas.quotaZookeeper+"/bug", null, null, -1, 1, 1, 1); - dserTree.createNode(Quotas.quotaPath("/bug"), new byte[20], null, -1, 1, 1, 1); + dserTree.createNode(Quotas.limitPath("/bug"), new byte[20], null, -1, 1, 1, 1); dserTree.createNode(Quotas.statPath("/bug"), new byte[20], null, -1, 1, 1, 1); //deserialize a DataTree; this should clear the old /bug nodes and pathTrie diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/DeserializationPerfTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/DeserializationPerfTest.java index b9f1df8cc4d..6f1b6aee7a6 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/DeserializationPerfTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/DeserializationPerfTest.java @@ -35,7 +35,7 @@ public class DeserializationPerfTest extends ZKTestCase { protected static final Logger LOG = LoggerFactory.getLogger(DeserializationPerfTest.class); private static void deserializeTree(int depth, int width, int len) - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { BinaryInputArchive ia; int count; { @@ -70,49 +70,49 @@ private static void deserializeTree(int depth, int width, int len) @Test public void testSingleDeserialize() throws - InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { deserializeTree(1, 0, 20); } @Test public void testWideDeserialize() throws - InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { deserializeTree(2, 10000, 20); } @Test public void testDeepDeserialize() throws - InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { deserializeTree(400, 1, 20); } @Test public void test10Wide5DeepDeserialize() throws - InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { deserializeTree(5, 10, 20); } @Test public void test15Wide5DeepDeserialize() throws - InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { deserializeTree(5, 15, 20); } @Test public void test25Wide4DeepDeserialize() throws - InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { deserializeTree(4, 25, 20); } @Test public void test40Wide4DeepDeserialize() throws - InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { deserializeTree(4, 40, 20); } @Test public void test300Wide3DeepDeserialize() throws - InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { deserializeTree(3, 300, 20); } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/SerializationPerfTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/SerializationPerfTest.java index 43982322dfe..1bf474be644 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/SerializationPerfTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/SerializationPerfTest.java @@ -38,7 +38,7 @@ public void write(int b) { } static int createNodes(DataTree tree, String path, int depth, - int childcount, int parentCVersion, byte[] data) throws KeeperException.NodeExistsException, KeeperException.NoNodeException { + int childcount, int parentCVersion, byte[] data) throws KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { path += "node" + depth; tree.createNode(path, data, null, -1, ++parentCVersion, 1, 1); @@ -57,7 +57,7 @@ static int createNodes(DataTree tree, String path, int depth, } private static void serializeTree(int depth, int width, int len) - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { DataTree tree = new DataTree(); createNodes(tree, "/", depth, width, tree.getNode("/").stat.getCversion(), new byte[len]); int count = tree.getNodeCount(); @@ -77,49 +77,49 @@ private static void serializeTree(int depth, int width, int len) @Test public void testSingleSerialize() - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { serializeTree(1, 0, 20); } @Test public void testWideSerialize() - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { serializeTree(2, 10000, 20); } @Test public void testDeepSerialize() - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { serializeTree(400, 1, 20); } @Test public void test10Wide5DeepSerialize() - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { serializeTree(5, 10, 20); } @Test public void test15Wide5DeepSerialize() - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { serializeTree(5, 15, 20); } @Test public void test25Wide4DeepSerialize() - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { serializeTree(4, 25, 20); } @Test public void test40Wide4DeepSerialize() - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { serializeTree(4, 40, 20); } @Test public void test300Wide3DeepSerialize() - throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException { + throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException, KeeperException.QuotaExceededException { serializeTree(3, 300, 20); } } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/FuzzySnapshotRelatedTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/FuzzySnapshotRelatedTest.java index 4005dbeb8eb..dd1a6324c2d 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/FuzzySnapshotRelatedTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/FuzzySnapshotRelatedTest.java @@ -31,6 +31,7 @@ import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.KeeperException.QuotaExceededException; import org.apache.zookeeper.Op; import org.apache.zookeeper.PortAssignment; @@ -345,7 +346,7 @@ public void addListener(String path, NodeSerializeListener listener) { public void createNode(final String path, byte data[], List acl, long ephemeralOwner, int parentCVersion, long zxid, long time, Stat outputStat) - throws NoNodeException, NodeExistsException { + throws NoNodeException, NodeExistsException, QuotaExceededException { NodeCreateListener listener = nodeCreateListeners.get(path); if (listener != null) { listener.process(path); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuotasTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuotasTest.java new file mode 100644 index 00000000000..59ae90fb9d1 --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuotasTest.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.test; + +import org.apache.zookeeper.Quotas; +import org.junit.Assert; +import org.junit.Test; + +public class QuotasTest { + + @Test + public void testStatPath() { + Assert.assertEquals("/zookeeper/quota/foo/zookeeper_stats", Quotas.statPath("/foo")); + } + + @Test + public void testLimitPath() { + Assert.assertEquals("/zookeeper/quota/foo/zookeeper_limits", Quotas.limitPath("/foo")); + } + + @Test + public void testQuotaPathPath() { + Assert.assertEquals("/zookeeper/quota/foo", Quotas.quotaPath("/foo")); + } + + @Test + public void testTrimQuotaPath() { + Assert.assertEquals("/foo", Quotas.trimQuotaPath("/zookeeper/quota/foo")); + } +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/StatsTrackTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/StatsTrackTest.java new file mode 100644 index 00000000000..16f792420f2 --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/StatsTrackTest.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.test; + +import org.apache.zookeeper.StatsTrack; +import org.junit.Assert; +import org.junit.Test; + +public class StatsTrackTest { + + public static class OldStatsTrack { + private int count; + private long bytes; + private String countStr = "count"; + private String byteStr = "bytes"; + + /** + * a default constructor for + * stats + */ + public OldStatsTrack() { + this(null); + } + /** + * the stat string should be of the form count=int,bytes=long + * if stats is called with null the count and bytes are initialized + * to -1. + * @param stats the stat string to be intialized with + */ + public OldStatsTrack(String stats) { + if (stats == null) { + stats = "count=-1,bytes=-1"; + } + String[] split = stats.split(","); + if (split.length != 2) { + throw new IllegalArgumentException("invalid string " + stats); + } + count = Integer.parseInt(split[0].split("=")[1]); + bytes = Long.parseLong(split[1].split("=")[1]); + } + + + /** + * get the count of nodes allowed as part of quota + * + * @return the count as part of this string + */ + public int getCount() { + return this.count; + } + + /** + * set the count for this stat tracker. + * + * @param count + * the count to set with + */ + public void setCount(int count) { + this.count = count; + } + + /** + * get the count of bytes allowed as part of quota + * + * @return the bytes as part of this string + */ + public long getBytes() { + return this.bytes; + } + + /** + * set teh bytes for this stat tracker. + * + * @param bytes + * the bytes to set with + */ + public void setBytes(long bytes) { + this.bytes = bytes; + } + + @Override + /* + * returns the string that maps to this stat tracking. + */ + public String toString() { + return countStr + "=" + count + "," + byteStr + "=" + bytes; + } + } + + @Test + public void testBackwardCompatibility() { + StatsTrack quota = new StatsTrack(null); + quota.setCount(4); + quota.setCountHardLimit(4); + quota.setBytes(9L); + quota.setByteHardLimit(15L); + + OldStatsTrack ost = new OldStatsTrack(quota.toString()); + Assert.assertTrue("bytes are set", ost.getBytes() == 9L); + Assert.assertTrue("num count is set", ost.getCount() == 4); + } + + @Test + public void testUpwardCompatibility() { + OldStatsTrack ost = new OldStatsTrack(null); + ost.setCount(2); + ost.setBytes(5); + + StatsTrack st = new StatsTrack(ost.toString()); + Assert.assertEquals(5, st.getBytes()); + Assert.assertEquals(2, st.getCount()); + Assert.assertEquals(-1, st.getByteHardLimit()); + Assert.assertEquals(-1, st.getCountHardLimit()); + } +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ZooKeeperQuotaTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ZooKeeperQuotaTest.java index d4d217de199..a885f3147f6 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ZooKeeperQuotaTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ZooKeeperQuotaTest.java @@ -18,25 +18,25 @@ package org.apache.zookeeper.test; -import java.io.IOException; - import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.QuotaExceededException; import org.apache.zookeeper.Quotas; import org.apache.zookeeper.StatsTrack; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooKeeperMain; import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.cli.DelQuotaCommand; +import org.apache.zookeeper.cli.SetQuotaCommand; import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.test.StatsTrackTest.OldStatsTrack; import org.junit.Assert; import org.junit.Test; public class ZooKeeperQuotaTest extends ClientBase { @Test - public void testQuota() throws IOException, - InterruptedException, KeeperException, Exception { + public void testQuota() throws Exception { final ZooKeeper zk = createClient(); final String path = "/a/b/v"; // making sure setdata works on / @@ -52,16 +52,30 @@ public void testQuota() throws IOException, zk.create("/a/b/v/d", "some".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - ZooKeeperMain.createQuota(zk, path, 5L, 10); + StatsTrack quota = new StatsTrack(null); + quota.setCount(4); + quota.setCountHardLimit(4); + quota.setBytes(9L); + quota.setByteHardLimit(15L); + SetQuotaCommand.createQuota(zk, path, quota); // see if its set - String absolutePath = Quotas.quotaZookeeper + path + "/" + Quotas.limitNode; + String absolutePath = Quotas.limitPath(path); byte[] data = zk.getData(absolutePath, false, new Stat()); StatsTrack st = new StatsTrack(new String(data)); - Assert.assertTrue("bytes are set", st.getBytes() == 5L); - Assert.assertTrue("num count is set", st.getCount() == 10); + Assert.assertTrue("bytes are set", st.getBytes() == 9L); + Assert.assertTrue("byte hard limit is set", + st.getByteHardLimit() == 15L); + Assert.assertTrue("num count is set", st.getCount() == 4); + Assert.assertTrue("count hard limit is set", + st.getCountHardLimit() == 4); + + // check quota node readable by old servers + OldStatsTrack ost = new OldStatsTrack(new String(data)); + Assert.assertTrue("bytes are set", ost.getBytes() == 9L); + Assert.assertTrue("num count is set", ost.getCount() == 4); - String statPath = Quotas.quotaZookeeper + path + "/" + Quotas.statNode; + String statPath = Quotas.statPath(path); byte[] qdata = zk.getData(statPath, false, new Stat()); StatsTrack qst = new StatsTrack(new String(qdata)); Assert.assertTrue("bytes are set", qst.getBytes() == 8L); @@ -76,4 +90,114 @@ public void testQuota() throws IOException, Assert.assertNotNull("Quota is still set", server.getZKDatabase().getDataTree().getMaxPrefixWithQuota(path) != null); } + + @Test + public void testSetQuotaExceedBytes() throws Exception { + + final ZooKeeper zk = createClient(); + final String path = "/c1"; + //zk.create("/test", null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + zk.create(path, "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + StatsTrack st = new StatsTrack(); + st.setByteHardLimit(5L); + SetQuotaCommand.createQuota(zk, path, st); + + try { + zk.setData(path, "newdata".getBytes(), -1); + } catch (QuotaExceededException e) { + //expected + } + + st = new StatsTrack(); + st.setByteHardLimit(1); + DelQuotaCommand.delQuota(zk, path, st); + st = new StatsTrack(); + st.setBytes(5L); + SetQuotaCommand.createQuota(zk, path, st); + zk.setData(path, "newdata".getBytes(), -1); + } + + @Test + public void testSetQuotaExceedCount() throws Exception { + + final ZooKeeper zk = createClient(); + final String path = "/c1"; + zk.create(path, "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + int count = 2; + StatsTrack st = new StatsTrack(); + st.setCountHardLimit(count); + SetQuotaCommand.createQuota(zk, path, st); + zk.create(path + "/c2", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + + try { + zk.create(path + "/c2" + "/c3", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } catch (QuotaExceededException e) { + //expected + } + + st = new StatsTrack(); + st.setCountHardLimit(1); + DelQuotaCommand.delQuota(zk, path, st); + st = new StatsTrack(); + st.setCount(count); + SetQuotaCommand.createQuota(zk, path, st); + zk.create(path + "/c2" + "/c3", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } + + @Test + public void testSetQuotaWhenChildExceedBytes() throws Exception { + + final ZooKeeper zk = createClient(); + final String path = "/test/quota"; + zk.create("/test", null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + zk.create("/test/quota", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + zk.create("/test/quota/data", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + ZooKeeperMain.createQuota(zk, path, 5L, 10); + try { + zk.setData("/test/quota/data", "newdata".getBytes(), -1); + } catch (QuotaExceededException e) { + //expected + } + } + + @Test + public void testSetQuotaWhenCreateNodesExceedBytes() throws Exception { + + final ZooKeeper zk = createClient(); + final String path = "/test/quota"; + zk.create("/test", null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + zk.create("/test/quota", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + ZooKeeperMain.createQuota(zk, path, 5L, 10); + try { + zk.create("/test/quota/data", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } catch (QuotaExceededException e) { + //expected + } + } + + @Test + public void testSetQuotaWhenCreateNodeExceedCount() throws Exception { + + final ZooKeeper zk = createClient(); + final String path = "/test/quota"; + zk.create("/test", null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + zk.create("/test/quota", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + ZooKeeperMain.createQuota(zk, path, 100L, 1); + try { + zk.create("/test/quota/data", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } catch (QuotaExceededException e) { + //expected + } + } }