Skip to content

Commit

Permalink
ENG-7609 merge from master
Browse files Browse the repository at this point in the history
  • Loading branch information
Steve Cooper committed Mar 24, 2015
2 parents d8ec100 + 13e5006 commit bdc6b2c
Show file tree
Hide file tree
Showing 133 changed files with 3,425 additions and 1,919 deletions.
4 changes: 2 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@ doc/tutorials/auction/csvloader_*.csv
/tests/ee/.project
/tests/sqlcmd/scripts/**/*.err
/tests/sqlcmd/scripts/**/*.errclean
/tests/sqlcmd/scripts/**/*.errdiff
/tests/sqlcmd/scripts/**/*.errdiffs
/tests/sqlcmd/scripts/**/*.out
/tests/sqlcmd/scripts/**/*.outclean
/tests/sqlcmd/scripts/**/*.outdiff
/tests/sqlcmd/scripts/**/*.outdiffs
/tools/voltify.venv
/src/ee/org_voltdb_utils_PosixAdvise.h
/src/frontend/org/voltcore/utils/DBBPool.java
Expand Down
10 changes: 10 additions & 0 deletions build-client.xml
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,16 @@ JAR BUILDING
</jar>
</target>

<target name="maven-jars"
description = "makes extra .jar files that get pushed to maven repo"
depends="stage_src, javadoc, buildinfo">
<jar destfile="${build.dir}/voltdbclient-${dist.version}-javadoc.jar"
basedir="${doc.dir}/javadoc/java-client-api"
/>
<jar destfile="${build.dir}/voltdbclient-${dist.version}-sources.jar"
basedir="${build.clientsrc.dir}"
/>
</target>

<!--
***************************************
Expand Down
Binary file modified doc/AdminGuide.pdf
Binary file not shown.
Binary file modified doc/UsingVoltDB.pdf
Binary file not shown.
22 changes: 22 additions & 0 deletions lib/python/voltcli/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -466,6 +466,28 @@ def find_resource(self, name, required=False):
utility.abort('Resource file "%s" is missing.' % name)
return None

def voltdb_connect(self, host, port, username=None, password=None):
"""
Create a VoltDB client connection.
"""
self.voltdb_disconnect()
try:
kwargs = {}
if username:
kwargs['username'] = username
if password:
kwargs['password'] = password
self.client = FastSerializer(host, port, **kwargs)
except Exception, e:
utility.abort(e)

def voltdb_disconnect(self):
"""
Close a VoltDB client connection.
"""
if self.client:
self.client.close()

def _print_verb_help(self, verb_name):
# Internal method to display help for a verb
verb = self.verbspace.verbs[verb_name]
Expand Down
20 changes: 8 additions & 12 deletions lib/python/voltcli/verbs.py
Original file line number Diff line number Diff line change
Expand Up @@ -453,15 +453,16 @@ def initialize(self, verb):
None))

def go(self, verb, runner):
final_args = None
if self.subcommand == 'create':
if runner.opts.replica:
self.subcommand = 'replica'
final_args = [self.subcommand, 'replica']
if self.supports_live:
if runner.opts.block:
final_args = [self.subcommand]
else:
final_args = ['live', self.subcommand]
else:
elif final_args == None:
final_args = [self.subcommand]
if self.safemode_available:
if runner.opts.safemode:
Expand Down Expand Up @@ -561,18 +562,13 @@ def __init__(self, default_port):
ConnectionBundle.__init__(self, default_port = default_port, min_count = 1, max_count = 1)

def start(self, verb, runner):
try:
kwargs = {}
if runner.opts.username:
kwargs['username'] = runner.opts.username
if runner.opts.password:
kwargs['password'] = runner.opts.password
runner.client = FastSerializer(runner.opts.host.host, runner.opts.host.port, **kwargs)
except Exception, e:
utility.abort(e)
runner.voltdb_connect(runner.opts.host.host,
runner.opts.host.port,
username=runner.opts.username,
password=runner.opts.password)

def stop(self, verb, runner):
runner.client.close()
runner.voltdb_disconnect()

#===============================================================================
class ClientBundle(BaseClientBundle):
Expand Down
118 changes: 118 additions & 0 deletions lib/python/voltcli/voltadmin.d/stop.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
# This file is part of VoltDB.

# Copyright (C) 2008-2015 VoltDB Inc.
#
# This file contains original code and/or modifications of original code.
# Any modifications made by VoltDB Inc. are licensed under the following
# terms and conditions:
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

# Stop a node. Written to easily support multiple, but configured for
# a single host for now.


class Host(dict):

def __init__(self, id, abort_func):
self.id = id
self.abort_func = abort_func

# Provide pseudo-attributes for dictionary items with error checking
def __getattr__(self, name):
try:
return self[name]
except IndexError:
self.abort_func('Attribute "%s" not present for host.' % name)


class Hosts(object):

def __init__(self, abort_func):
self.hosts_by_id = {}
self.abort_func = abort_func

def update(self, host_id_raw, prop_name_raw, value_raw):
host_id = int(host_id_raw)
prop_name = prop_name_raw.lower()
value = value_raw
if prop_name.endswith('port'):
value = int(value)
self.hosts_by_id.setdefault(host_id, Host(host_id, self.abort_func))[prop_name] = value

def get_target_and_connection_host(self, host_name):
"""
Find an arbitrary host that isn't the one being stopped.
Returns a tuple with connection and target host objects.
"""
connection_host = None
target_host = None
for host in self.hosts_by_id.values():
if host.hostname == host_name:
target_host = host
elif connection_host is None:
connection_host = host
if not connection_host is None and not target_host is None:
break
return (target_host, connection_host)


@VOLT.Command(
bundles = VOLT.AdminBundle(),
description = 'Stop one host of a running VoltDB cluster.',
arguments = (
VOLT.StringArgument('target_host', 'the target HOST name or address'),
),
)
def stop(runner):

# Exec @SystemInformation to find out about the cluster.
response = runner.call_proc('@SystemInformation',
[VOLT.FastSerializer.VOLTTYPE_STRING],
['OVERVIEW'])

# Convert @SystemInformation results to objects.
hosts = Hosts(runner.abort)
for tuple in response.table(0).tuples():
hosts.update(tuple[0], tuple[1], tuple[2])

# Connect to an arbitrary host that isn't being stopped.
(thost, chost) = hosts.get_target_and_connection_host(runner.opts.target_host)
if thost is None:
runner.abort('Host not found in cluster: %s' % runner.opts.target_host)
if chost is None:
runner.abort('The entire cluster is being stopped, use "shutdown" instead.')

if runner.opts.username:
user_info = ', user: %s' % runner.opts.username
else:
user_info = ''
runner.info('Connecting to host: %s:%d%s' % (chost.hostname, chost.adminport, user_info))
runner.voltdb_connect(chost.hostname, chost.adminport,
runner.opts.username, runner.opts.password)

# Stop the requested host using exec @StopNode HOST_ID
runner.info('Stopping host %d: %s' % (thost.id, thost.hostname))
if not runner.opts.dryrun:
response = runner.call_proc('@StopNode',
[VOLT.FastSerializer.VOLTTYPE_INTEGER],
[thost.id],
check_status=False)
print response
89 changes: 74 additions & 15 deletions src/catgen/in/javasrc/CatalogDiffEngine.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,16 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;

import org.apache.commons.lang3.StringUtils;
import org.voltdb.VoltType;
import org.voltdb.catalog.CatalogType;
import org.voltdb.catalog.Cluster;
import org.voltdb.catalog.ConnectorProperty;
import org.voltdb.catalog.ConnectorTableInfo;
import org.voltdb.catalog.CatalogChangeGroup.FieldChange;
import org.voltdb.catalog.CatalogChangeGroup.TypeChanges;
import org.voltdb.catalog.Column;
Expand Down Expand Up @@ -68,7 +72,11 @@ public class CatalogDiffEngine {
// while no snapshot is running
private boolean m_requiresSnapshotIsolation = false;

private SortedMap<String,String> m_tablesThatMustBeEmpty = new TreeMap<>();
private final SortedMap<String,String> m_tablesThatMustBeEmpty = new TreeMap<>();

//Track new tables to help determine which export table is new or
//modified
private final SortedSet<String> m_newTablesForExport = new TreeSet<>();

//A very rough guess at whether only deployment changes are in the catalog update
//Can be improved as more deployment things are going to be allowed to conflict
Expand Down Expand Up @@ -306,6 +314,23 @@ static private boolean isTableLimitDeleteStmt(final CatalogType catType) {
return false;
}

/**
* If it is not a new table make sure that the soon to be exported
* table is empty or has no tuple allocated memory associated with it
*
* @param tName table name
*/
private void trackExportOfAlreadyExistingTables(String tName) {
if (tName == null || tName.trim().isEmpty()) return;
if (!m_newTablesForExport.contains(tName)) {
String errorMessage = String.format(
"Unable to change table %s to an export table because the table is not empty",
tName
);
m_tablesThatMustBeEmpty.put(tName, errorMessage);
}
}

/**
* @return null if the CatalogType can be dynamically added or removed
* from a running system. Return an error string if it can't be changed on
Expand All @@ -322,7 +347,6 @@ private String checkAddDropWhitelist(final CatalogType suspect, final ChangeType
if (suspect instanceof User ||
suspect instanceof Group ||
suspect instanceof Procedure ||
suspect instanceof Connector ||
suspect instanceof SnapshotSchedule ||
// refs are safe to add drop if the thing they reference is
suspect instanceof ConstraintRef ||
Expand All @@ -334,13 +358,45 @@ private String checkAddDropWhitelist(final CatalogType suspect, final ChangeType
// So, in short, all of these constraints will pass or fail tests of other catalog differences
// Even if they did show up as Constraints in the catalog (for no apparent functional reason),
// flagging their changes here would be redundant.
suspect instanceof Constraint ||
// Support add/drop of the top level object.
suspect instanceof Table)
suspect instanceof Constraint)
{
return null;
}

else if (suspect instanceof Table) {
Table tbl = (Table)suspect;
if ( ChangeType.ADDITION == changeType
&& CatalogUtil.isTableExportOnly((Database)tbl.getParent(), tbl)
) {
m_newTablesForExport.add(tbl.getTypeName());
}
// Support add/drop of the top level object.
return null;
}

else if (suspect instanceof Connector) {
if (ChangeType.ADDITION == changeType) {
for (ConnectorTableInfo cti: ((Connector)suspect).getTableinfo()) {
if (cti.getTable().getIsdred()) {
return "May not export a DR table";
}
trackExportOfAlreadyExistingTables(cti.getTable().getTypeName());
}
}
return null;
}

else if (suspect instanceof ConnectorTableInfo) {
if (ChangeType.ADDITION == changeType) {
trackExportOfAlreadyExistingTables(((ConnectorTableInfo)suspect).getTable().getTypeName());
}
return null;
}

else if (suspect instanceof ConnectorProperty) {
return null;
}

else if (suspect instanceof ColumnRef) {
if (suspect.getParent() instanceof Index) {
Index parent = (Index) suspect.getParent();
Expand Down Expand Up @@ -373,6 +429,9 @@ else if (suspect instanceof Column) {
if (CatalogUtil.isTableExportOnly((Database)table.getParent(), table)) {
return "May not dynamically add, drop, or rename export table columns.";
}
if (table.getIsdred()) {
return "May not dynamically add, drop, or rename DR table columns.";
}
if (changeType == ChangeType.ADDITION) {
Column col = (Column) suspect;
if ((! col.getNullable()) && (col.getDefaultvalue() == null)) {
Expand Down Expand Up @@ -483,6 +542,9 @@ private String[] checkAddDropIfTableIsEmptyWhitelist(final CatalogType suspect,
if (CatalogUtil.isTableExportOnly((Database)table.getParent(), table)) {
return null;
}
if (table.getIsdred()) {
return null;
}
retval[0] = parent.getTypeName();
retval[1] = String.format(
"Unable to add NOT NULL column %s because table %s is not empty and no default value was specified.",
Expand Down Expand Up @@ -580,8 +642,7 @@ private String checkModifyWhitelist(final CatalogType suspect,
// cases of BEFORE and AFTER values by listing the offending values.
String restrictionQualifier = "";

if (suspect instanceof Cluster && field.equals("drClusterId") ||
suspect instanceof Cluster && field.equals("drProducerPort")) {
if (suspect instanceof Cluster && field.equals("drProducerPort")) {
// Don't allow changes to ClusterId or ProducerPort while not transitioning to or from Disabled
if ((Boolean)prevType.getField("drProducerEnabled") && (Boolean)suspect.getField("drProducerEnabled")) {
restrictionQualifier = " while DR is enabled";
Expand All @@ -590,14 +651,6 @@ private String checkModifyWhitelist(final CatalogType suspect,
return null;
}
}
if (suspect instanceof Cluster && field.equals("drMasterHost")) {
if ((Boolean)suspect.getField("drProducerEnabled")) {
restrictionQualifier = " active-active and daisy-chained DR unsupported";
}
else {
return null;
}
}
if (suspect instanceof Constraint && field.equals("index"))
return null;
if (suspect instanceof Table) {
Expand Down Expand Up @@ -629,6 +682,9 @@ private String checkModifyWhitelist(final CatalogType suspect,
if (CatalogUtil.isTableExportOnly((Database)table.getParent(), table)) {
return "May not dynamically change the columns of export tables.";
}
if (table.getIsdred()) {
return "May not dynamically modify DR table columns.";
}

if (field.equals("index")) {
return null;
Expand Down Expand Up @@ -800,6 +856,9 @@ public String[] checkModifyIfTableIsEmptyWhitelist(final CatalogType suspect,
if (CatalogUtil.isTableExportOnly(db, table)) {
return null;
}
if (table.getIsdred()) {
return null;
}

// capture the table name
retval[0] = table.getTypeName();
Expand Down
Loading

0 comments on commit bdc6b2c

Please sign in to comment.