From 3944a3c16f8ca614674b007f4922dcccdcc3e60f Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 18 Jun 2013 17:15:11 -0700 Subject: [PATCH] Cleanup the arguments and add sanity check --- bin/generate_cluster_xml.py | 62 +++++++++++++++++++--------- config/tools/dummy-stores.xml | 77 ----------------------------------- 2 files changed, 42 insertions(+), 97 deletions(-) delete mode 100644 config/tools/dummy-stores.xml diff --git a/bin/generate_cluster_xml.py b/bin/generate_cluster_xml.py index 323be25f1a..1cbf74aa7b 100644 --- a/bin/generate_cluster_xml.py +++ b/bin/generate_cluster_xml.py @@ -26,16 +26,25 @@ # python generate_cluster_xml.py --file # --name # --nodes -# --partitions +# --partitions # --sock-port # --admin-port # --http-port -# --voldemort_home -# --current-stores +# --current-stores # --output-dir # --zones # -# The non zoned would look similar with the exception of the absence of the --zones switch. +# For non zoned cluster use : +# python generate_cluster_xml.py --file +# --name +# --nodes +# --partitions +# --sock-port +# --admin-port +# --http-port +# --current-stores +# --output-dir +# Note the absence of the --zones switch import sys import random @@ -48,6 +57,7 @@ except ImportError: print "Python 2.7 or higher is needed" + # Get a random seed rseed = int(random.randint(00000000001,99999999999)) @@ -58,7 +68,7 @@ help='the file of the list of hosts(one per line)') parser.add_argument('-N', '--name', type=str, default='voldemort', dest='name', help='the name you want to give the cluster') -parser.add_argument('-n', '--nodes', type=int, default=5, dest='nodes', +parser.add_argument('-n', '--nodes', type=int, default=6, dest='nodes', help='the number of nodes in the cluster') parser.add_argument('-p', '--partitions', type=int, default=1500, dest='partitions', help='number of partitions') @@ -68,15 +78,15 @@ dest='admin_port', help='admin port number') parser.add_argument('-hp', '--http-port', type=int, default=6665, dest='http_port', help='http port number') -parser.add_argument('-v', '--voldemort-home', type=str, dest='vold_home', - help='Path to voldemort home') -parser.add_argument('-s', '--current-stores', type=str, dest='current_stores', +parser.add_argument('-s', '--current-stores', type=str, default= "config/tools/dummy-stores-3zoned.xml", + dest='current_stores', help='Path to current stores xml. If you do not have info about the stores yet' 'use config/tools/dummy-stores.xml from the root voldemort home folder.') parser.add_argument('-o', '--output-dir', type=str, dest='output_dir', help='output directory location') parser.add_argument('-z', '--zones', type=int, dest='zones', - help='the number of zones you will have') + help='For non zoned clusters do not provide this argument.' + 'For zoned clusters provide this argument with at least two zones.') genType = parser.add_mutually_exclusive_group() genType.add_argument('--seed', type=int, default=rseed, dest='seed', @@ -105,6 +115,10 @@ # TODO : It would be ideal to have the script accept a list of zone ids. if args.zones: zones = args.zones + if (zones == 1): + print "For non zoned clusters do not provide this argument." + print "For zoned clusters provide this argument with at least two zones." + sys.exit(1) if (args.nodes % zones) != 0: print "Number of nodes must be evenly divisible by number of zones" sys.exit(1) @@ -121,14 +135,17 @@ sock_port = args.sock_port admin_port = args.admin_port seed = args.seed -current_stores = args.current_stores -vold_home = args.vold_home +vold_home = os.pardir +current_stores = os.path.join(vold_home, args.current_stores); # Generate the full list of partition IDs part_ids = range(partitions) if part_ids < 1500: - print "Warning : The number of partitions seems to be low. Recommended value is 1500 or more" - + print 'Warning : The number of partitions seems to be low. Assuming max of 3 zones and 50 nodes ' \ + 'per zone, a partition value of 1500 is recommended as it ensures an average of 10 ' \ + 'partitions per node.' + print 'Warning : The number of partitions seems to be low. Recommended value is 1500 or more.' + # Generate full list of zone IDs if args.zones: zone_ids = range(zones) @@ -152,13 +169,18 @@ print >> fileHandle, " %s" % str(proximityList).strip('[]') print >> fileHandle, " " -# TODO : Currently, random partitions are assigned to the nodes. A better approach would be to -# have some intelligence in the allocation such that consecutive partition-ids do not land -# on the same node. +# TODO : Currently, random partitions are assigned to the nodes in a round robine fashion. +# A better approach would be to have some intelligence in the allocation such that +# consecutive partition-ids do not land on the same node. for i in xrange(nodes): - node_partitions = ", ".join(str(p) for p in sorted(part_ids[i*(partitions/nodes):(i+1)*(partitions/nodes)])) - + j = i + node_partitions = list() + while j < len(part_ids): + node_partitions.append(str(part_ids[j])) + j += nodes; + partitionslist = ", ".join(node_partitions); + print >> fileHandle, " " print >> fileHandle, " %d" % i if args.file: @@ -168,7 +190,7 @@ print >> fileHandle, " %d" % http_port print >> fileHandle, " %d" % sock_port print >> fileHandle, " %d" % admin_port - print >> fileHandle, " %s" % node_partitions + print >> fileHandle, " %s" % partitionslist # If zones are being used, assign a zone-id if args.zones: print >> fileHandle, " %d" % zone_id @@ -181,7 +203,7 @@ fileHandle.close() -# For zones clusters call rebalance-new-cluster.sh +# For zoned clusters call rebalance-new-cluster.sh if args.zones: scriptPath = vold_home + '/bin/rebalance-new-cluster.sh' cmd = [scriptPath, '-v', vold_home, '-c', clusterXMLFilePath, '-s', current_stores, diff --git a/config/tools/dummy-stores.xml b/config/tools/dummy-stores.xml deleted file mode 100644 index 93dedc7b77..0000000000 --- a/config/tools/dummy-stores.xml +++ /dev/null @@ -1,77 +0,0 @@ - - - dummy_322_one - bdb - zone-routing - client - 6 - - 3 - 3 - - 2 - 2 - 0 - 0 - proximity-handoff - - json - "string" - - - json - [{'opt_out_list':'string'}] - - - - dummy_322_two - bdb - zone-routing - client - 6 - - 3 - 3 - - 2 - 2 - 0 - 0 - proximity-handoff - - json - "string" - - - json - [{'opt_out_list':'string'}] - - - - dummy_211_one - bdb - zone-routing - client - 4 - - 2 - 2 - - 2 - 1 - 1 - 1 - 0 - 0 - proximity-handoff - - string - UTF-8 - - - string - UTF-8 - - 14 - -