Skip to content
This repository has been archived by the owner on Feb 12, 2022. It is now read-only.

Commit

Permalink
Merge branch 'master' of git://github.com/tmaurel/scoot into tmaurel-…
Browse files Browse the repository at this point in the history 
…master
  • Loading branch information
@ivarley
ivarley committed Jan 3, 2013
2 parents 0fe1cc8 + 17d5289 commit d408392
Show file tree
Hide file tree
Showing 9 changed files with 98 additions and 13 deletions.
2 changes: 2 additions & 0 deletions src/main/java/com/salesforce/scoot/parser/HBaseScootXMLParser.java
View file
Expand Up @@ -79,6 +79,8 @@ public class HBaseScootXMLParser extends HBaseSchemaParser {
propertyNames.put("blockSizeKB", HBaseSchemaAttribute.BLOCKSIZE.name());
propertyNames.put("bloomFilter", HBaseSchemaAttribute.BLOOMFILTER.name());
propertyNames.put("inMemory", HBaseSchemaAttribute.IN_MEMORY.name());
propertyNames.put("keepDeletedCells", HBaseSchemaAttribute.KEEP_DELETED_CELLS.name());
propertyNames.put("minVersions", HBaseSchemaAttribute.MIN_VERSIONS.name());
propertyNames.put("maxVersions", HBaseSchemaAttribute.VERSIONS.name());
propertyNames.put("replicationScope", HBaseSchemaAttribute.REPLICATION_SCOPE.name());
propertyNames.put("timeToLiveMS", HBaseSchemaAttribute.TTL.name());
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/com/salesforce/scoot/scripter/HBaseRubySchemaPatchScripter.java
View file
Expand Up @@ -225,7 +225,8 @@ private void scriptVerifyTableMatches(HTableDescriptor oldTable, String operatio
// now descend into child objects
for (HColumnDescriptor c : oldTable.getColumnFamilies()){
s(" # Column family: " + c.getNameAsString());
s(" cf = HColumnDescriptor.new(\"" + c.getNameAsString() + "\")");
s(" cfname = \"" + c.getNameAsString() + "\"");
s(" cf = table.getFamily(cfname.bytes.to_a)");

for (Entry<String,String> p : getSortedStringEntries(c.getValues())){
s(" compare(" + errorCollectionName + ", cf, \"" + operationName + "\", \"" + p.getKey() + "\", \"" + escapeDoubleQuotes(p.getValue()) + "\")");
Expand Down
48 changes: 47 additions & 1 deletion src/test/java/com/salesforce/scoot/ScootTest.java
View file
Expand Up @@ -27,10 +27,18 @@

import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import org.apache.hadoop.hbase.util.Bytes;
import java.net.URL;
import java.util.List;

import junit.framework.TestCase;

import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.util.Bytes;

import com.google.common.io.Resources;
import com.salesforce.scoot.parser.HBaseScootXMLParser;

/**
* General tests for scoot functionality
*
Expand Down Expand Up @@ -65,4 +73,42 @@ public void testHelp() throws Exception {
}

}

public void testParse() throws Exception {
URL xmlFile = Resources.getResource("ScootXMLParserTest.xml");
String fileName = xmlFile.getFile();
HBaseScootXMLParser parser = new HBaseScootXMLParser();
parser.setResourceToParse(fileName);
HBaseSchema schema = parser.parse();

List<HTableDescriptor> tables = schema.getTables();
assertEquals(1, tables.size());

// Test table attributes
HTableDescriptor table = tables.get(0);
assertEquals("testMe", table.getNameAsString());
assertEquals("536870912", table.getValue(HBaseSchemaAttribute.MAX_FILESIZE.name()));
assertEquals("true", table.getValue(HBaseSchemaAttribute.READONLY.name()));
assertEquals("128974848", table.getValue(HBaseSchemaAttribute.MEMSTORE_FLUSHSIZE.name()));
assertEquals("true", table.getValue(HBaseSchemaAttribute.DEFERRED_LOG_FLUSH.name()));
assertEquals("ivarley", table.getOwnerString());

HColumnDescriptor[] columnFamilies = table.getColumnFamilies();
assertNotNull(columnFamilies);
assertEquals(1, columnFamilies.length);

// Test column family attributes
HColumnDescriptor columnFamily = columnFamilies[0];
assertEquals("testMeColumnFamily1", columnFamily.getNameAsString());
assertEquals("10", columnFamily.getValue(HBaseSchemaAttribute.VERSIONS.name()));
assertEquals("32768", columnFamily.getValue(HBaseSchemaAttribute.BLOCKSIZE.name()));
assertEquals("false", columnFamily.getValue(HBaseSchemaAttribute.BLOCKCACHE.name()));
assertEquals("123456789", columnFamily.getValue(HBaseSchemaAttribute.TTL.name()));
assertEquals("true", columnFamily.getValue(HBaseSchemaAttribute.IN_MEMORY.name()));
assertEquals("ROW", columnFamily.getValue(HBaseSchemaAttribute.BLOOMFILTER.name()));
assertEquals("true", columnFamily.getValue(HBaseSchemaAttribute.KEEP_DELETED_CELLS.name()));
assertEquals("5", columnFamily.getValue(HBaseSchemaAttribute.MIN_VERSIONS.name()));
assertEquals("1", columnFamily.getValue(HBaseSchemaAttribute.REPLICATION_SCOPE.name()));

}
}
12 changes: 8 additions & 4 deletions src/test/resources/DiffScriptGenerationTestResultAB.rb
View file
Expand Up @@ -75,7 +75,8 @@ def compare(errs, obj, action, attr, val)
compare(preWarnings, table, "drop", "READONLY", "false")
compare(preWarnings, table, "drop", "fullSchema", "<table isReadOnly=\"false\" maxFileSizeMB=\"256\" memStoreFlushSizeMB=\"64\" name=\"dropMe\" owner=\"ivarley\" useDeferredLogFlush=\"false\"><key><keyPart inverted=\"false\" length=\"15\" name=\"dropMeKeyPart1\" type=\"String\"/><keyPart inverted=\"true\" length=\"15\" name=\"dropMeKeyPart2\" type=\"Timestamp\"/></key><columnFamilies><columnFamily blockCache=\"true\" blockSizeKB=\"64\" bloomFilter=\"NONE\" inMemory=\"false\" maxVersions=\"3\" name=\"dropMeColumnFamily1\" replicationScope=\"0\" timeToLiveMS=\"2147483647\"><column name=\"dropMeColumn1\" type=\"String\"/><column name=\"dropMeColumn2\" type=\"Timestamp\"/><column name=\"dropMeColumn3\" type=\"Byte\"/></columnFamily></columnFamilies></table>")
# Column family: dropMeColumnFamily1
cf = HColumnDescriptor.new("dropMeColumnFamily1")
cfname = "dropMeColumnFamily1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preWarnings, cf, "drop", "BLOCKCACHE", "true")
compare(preWarnings, cf, "drop", "BLOCKSIZE", "65536")
compare(preWarnings, cf, "drop", "BLOOMFILTER", "NONE")
Expand Down Expand Up @@ -114,7 +115,8 @@ def compare(errs, obj, action, attr, val)
compare(preErrors, table, "alter", "READONLY", "false")
compare(preErrors, table, "alter", "fullSchema", "<table isReadOnly=\"false\" maxFileSizeMB=\"256\" memStoreFlushSizeMB=\"64\" name=\"alterMe\" owner=\"ivarley\" useDeferredLogFlush=\"false\"><key><keyPart inverted=\"false\" length=\"15\" name=\"alterMeKeyPart1\" type=\"String\"/><keyPart inverted=\"true\" length=\"15\" name=\"alterMeKeyPart2\" type=\"Timestamp\"/></key><columnFamilies><columnFamily blockCache=\"true\" blockSizeKB=\"64\" bloomFilter=\"NONE\" inMemory=\"false\" maxVersions=\"3\" name=\"alterMeColumnFamily1\" replicationScope=\"0\" timeToLiveMS=\"2147483647\"><column name=\"alterMeColumn1\" type=\"String\"/><column name=\"alterMeColumn2\" type=\"Timestamp\"/><column name=\"alterMeColumn3\" type=\"Byte\"/></columnFamily></columnFamilies></table>")
# Column family: alterMeColumnFamily1
cf = HColumnDescriptor.new("alterMeColumnFamily1")
cfname = "alterMeColumnFamily1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preErrors, cf, "alter", "BLOCKCACHE", "true")
compare(preErrors, cf, "alter", "BLOCKSIZE", "65536")
compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE")
Expand Down Expand Up @@ -258,7 +260,8 @@ def compare(errs, obj, action, attr, val)
compare(preErrors, table, "create", "READONLY", "false")
compare(preErrors, table, "create", "fullSchema", "<table isReadOnly=\"false\" maxFileSizeMB=\"256\" memStoreFlushSizeMB=\"64\" name=\"createMe\" owner=\"ivarley\" useDeferredLogFlush=\"false\"><key><keyPart inverted=\"false\" length=\"15\" name=\"createMeKeyPart1\" type=\"String\"/><keyPart inverted=\"true\" length=\"15\" name=\"createMeKeyPart2\" type=\"Timestamp\"/></key><columnFamilies><columnFamily blockCache=\"true\" blockSizeKB=\"64\" bloomFilter=\"NONE\" inMemory=\"false\" maxVersions=\"3\" name=\"createMeColumnFamily1\" replicationScope=\"0\" timeToLiveMS=\"2147483647\"><column name=\"createMeColumn1\" type=\"String\"/><column name=\"createMeColumn2\" type=\"Timestamp\"/><column name=\"createMeColumn3\" type=\"Byte\"/></columnFamily></columnFamilies></table>")
# Column family: createMeColumnFamily1
cf = HColumnDescriptor.new("createMeColumnFamily1")
cfname = "createMeColumnFamily1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preErrors, cf, "create", "BLOCKCACHE", "true")
compare(preErrors, cf, "create", "BLOCKSIZE", "65536")
compare(preErrors, cf, "create", "BLOOMFILTER", "NONE")
Expand Down Expand Up @@ -291,7 +294,8 @@ def compare(errs, obj, action, attr, val)
compare(preErrors, table, "alter", "READONLY", "false")
compare(preErrors, table, "alter", "fullSchema", "<table isReadOnly=\"false\" maxFileSizeMB=\"257\" memStoreFlushSizeMB=\"65\" name=\"alterMe\" owner=\"ivarley2\" useDeferredLogFlush=\"false\"><key><keyPart inverted=\"false\" length=\"15\" name=\"alterMeKeyPart1\" type=\"String\"/><keyPart inverted=\"true\" length=\"15\" name=\"alterMeKeyPart2\" type=\"Timestamp\"/></key><columnFamilies><columnFamily blockCache=\"true\" blockSizeKB=\"65\" bloomFilter=\"NONE\" inMemory=\"false\" maxVersions=\"3\" name=\"alterMeColumnFamily1\" replicationScope=\"0\" timeToLiveMS=\"2147483647\"><column name=\"alterMeColumn1\" type=\"String\"/><column name=\"alterMeColumn2\" type=\"Timestamp\"/><column name=\"alterMeColumn3\" type=\"Byte\"/></columnFamily></columnFamilies></table>")
# Column family: alterMeColumnFamily1
cf = HColumnDescriptor.new("alterMeColumnFamily1")
cfname = "alterMeColumnFamily1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preErrors, cf, "alter", "BLOCKCACHE", "true")
compare(preErrors, cf, "alter", "BLOCKSIZE", "66560")
compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE")
Expand Down
3 changes: 2 additions & 1 deletion src/test/resources/DiffScriptGenerationTestResultD.rb
View file
Expand Up @@ -67,7 +67,8 @@ def compare(errs, obj, action, attr, val)
compare(preWarnings, table, "drop", "READONLY", "false")
compare(preWarnings, table, "drop", "fullSchema", "<table isReadOnly=\"false\" maxFileSizeMB=\"256\" memStoreFlushSizeMB=\"64\" name=\"createMe\" owner=\"ivarley\" useDeferredLogFlush=\"false\"><key><keyPart inverted=\"false\" length=\"15\" name=\"createMeKeyPart1\" type=\"String\"/><keyPart inverted=\"true\" length=\"15\" name=\"createMeKeyPart2\" type=\"Timestamp\"/></key><columnFamilies><columnFamily blockCache=\"true\" blockSizeKB=\"64\" bloomFilter=\"NONE\" inMemory=\"false\" maxVersions=\"3\" name=\"createMeColumnFamily1\" replicationScope=\"0\" timeToLiveMS=\"2147483647\"><column name=\"createMeColumn1\" type=\"String\"/><column name=\"createMeColumn2\" type=\"Timestamp\"/><column name=\"createMeColumn3\" type=\"Byte\"/></columnFamily></columnFamilies></table>")
# Column family: createMeColumnFamily1
cf = HColumnDescriptor.new("createMeColumnFamily1")
cfname = "createMeColumnFamily1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preWarnings, cf, "drop", "BLOCKCACHE", "true")
compare(preWarnings, cf, "drop", "BLOCKSIZE", "65536")
compare(preWarnings, cf, "drop", "BLOOMFILTER", "NONE")
Expand Down
6 changes: 4 additions & 2 deletions src/test/resources/DiffScriptGenerationTestResultEF.rb
View file
Expand Up @@ -67,7 +67,8 @@ def compare(errs, obj, action, attr, val)
compare(preErrors, table, "alter", "READONLY", "false")
compare(preErrors, table, "alter", "fullSchema", "<table name=\"minimal\"><columnFamilies><columnFamily name=\"minimalColumnFamily1\"></columnFamily></columnFamilies></table>")
# Column family: minimalColumnFamily1
cf = HColumnDescriptor.new("minimalColumnFamily1")
cfname = "minimalColumnFamily1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preErrors, cf, "alter", "BLOCKCACHE", "true")
compare(preErrors, cf, "alter", "BLOCKSIZE", "65536")
compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE")
Expand Down Expand Up @@ -160,7 +161,8 @@ def compare(errs, obj, action, attr, val)
compare(preErrors, table, "alter", "READONLY", "false")
compare(preErrors, table, "alter", "fullSchema", "<table isReadOnly=\"false\" maxFileSizeMB=\"10240\" memStoreFlushSizeMB=\"128\" name=\"minimal\" useDeferredLogFlush=\"false\"><columnFamilies><columnFamily blockCache=\"true\" blockSizeKB=\"64\" bloomFilter=\"NONE\" inMemory=\"false\" maxVersions=\"3\" name=\"minimalColumnFamily1\" replicationScope=\"0\" timeToLiveMS=\"2147483647\"></columnFamily></columnFamilies></table>")
# Column family: minimalColumnFamily1
cf = HColumnDescriptor.new("minimalColumnFamily1")
cfname = "minimalColumnFamily1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preErrors, cf, "alter", "BLOCKCACHE", "true")
compare(preErrors, cf, "alter", "BLOCKSIZE", "65536")
compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE")
Expand Down
6 changes: 4 additions & 2 deletions src/test/resources/DiffScriptGenerationTestResultFE.rb
View file
Expand Up @@ -67,7 +67,8 @@ def compare(errs, obj, action, attr, val)
compare(preErrors, table, "alter", "READONLY", "false")
compare(preErrors, table, "alter", "fullSchema", "<table isReadOnly=\"false\" maxFileSizeMB=\"10240\" memStoreFlushSizeMB=\"128\" name=\"minimal\" useDeferredLogFlush=\"false\"><columnFamilies><columnFamily blockCache=\"true\" blockSizeKB=\"64\" bloomFilter=\"NONE\" inMemory=\"false\" maxVersions=\"3\" name=\"minimalColumnFamily1\" replicationScope=\"0\" timeToLiveMS=\"2147483647\"></columnFamily></columnFamilies></table>")
# Column family: minimalColumnFamily1
cf = HColumnDescriptor.new("minimalColumnFamily1")
cfname = "minimalColumnFamily1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preErrors, cf, "alter", "BLOCKCACHE", "true")
compare(preErrors, cf, "alter", "BLOCKSIZE", "65536")
compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE")
Expand Down Expand Up @@ -160,7 +161,8 @@ def compare(errs, obj, action, attr, val)
compare(preErrors, table, "alter", "READONLY", "false")
compare(preErrors, table, "alter", "fullSchema", "<table name=\"minimal\"><columnFamilies><columnFamily name=\"minimalColumnFamily1\"></columnFamily></columnFamilies></table>")
# Column family: minimalColumnFamily1
cf = HColumnDescriptor.new("minimalColumnFamily1")
cfname = "minimalColumnFamily1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preErrors, cf, "alter", "BLOCKCACHE", "true")
compare(preErrors, cf, "alter", "BLOCKSIZE", "65536")
compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE")
Expand Down
6 changes: 4 additions & 2 deletions src/test/resources/PhoenixScriptGenerationTestResultA.rb
View file
Expand Up @@ -142,7 +142,8 @@ def compare(errs, obj, action, attr, val)
compare(preErrors, table, "create", "MEMSTORE_FLUSHSIZE", "134217728")
compare(preErrors, table, "create", "READONLY", "false")
# Column family: 1
cf = HColumnDescriptor.new("1")
cfname = "1"
cf = table.getFamily(cfname.bytes.to_a)
compare(preErrors, cf, "create", "BLOCKCACHE", "true")
compare(preErrors, cf, "create", "BLOCKSIZE", "65536")
compare(preErrors, cf, "create", "BLOOMFILTER", "NONE")
Expand All @@ -156,7 +157,8 @@ def compare(errs, obj, action, attr, val)
compare(preErrors, cf, "create", "TTL", "2147483647")
compare(preErrors, cf, "create", "VERSIONS", "3")
# Column family: 2
cf = HColumnDescriptor.new("2")
cfname = "2"
cf = table.getFamily(cfname.bytes.to_a)
compare(preErrors, cf, "create", "BLOCKCACHE", "true")
compare(preErrors, cf, "create", "BLOCKSIZE", "65536")
compare(preErrors, cf, "create", "BLOOMFILTER", "NONE")
Expand Down
25 changes: 25 additions & 0 deletions src/test/resources/ScootXMLParserTest.xml
View file
@@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<schema>

<table name="testMe"
maxFileSizeMB="512"
isReadOnly="true"
memStoreFlushSizeMB="123"
useDeferredLogFlush="true"
owner="ivarley">

<columnFamilies>
<columnFamily name="testMeColumnFamily1"
maxVersions="10"
blockSizeKB="32"
blockCache="false"
timeToLiveMS="123456789"
inMemory="true"
bloomFilter="ROW"
keepDeletedCells="true"
minVersions="5"
replicationScope="1" />
</columnFamilies>
</table>

</schema>

0 comments on commit d408392

Please sign in to comment.