From ecf98ca6c318b28a524f43eafa7b791987a56301 Mon Sep 17 00:00:00 2001 From: Thomas Maurel Date: Wed, 21 Nov 2012 23:14:45 +0100 Subject: [PATCH 1/2] Included keepDeletedCells and minVersions into HBaseScootXMLParser parsed attributes for column families. Added a small unit test (ScootTest#testParse) to make sure tables and column families attributes are parsed correctly. --- .../scoot/parser/HBaseScootXMLParser.java | 2 + .../java/com/salesforce/scoot/ScootTest.java | 48 ++++++++++++++++++- src/test/resources/ScootXMLParserTest.xml | 25 ++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/ScootXMLParserTest.xml diff --git a/src/main/java/com/salesforce/scoot/parser/HBaseScootXMLParser.java b/src/main/java/com/salesforce/scoot/parser/HBaseScootXMLParser.java index 3da4410..5472104 100644 --- a/src/main/java/com/salesforce/scoot/parser/HBaseScootXMLParser.java +++ b/src/main/java/com/salesforce/scoot/parser/HBaseScootXMLParser.java @@ -78,6 +78,8 @@ public class HBaseScootXMLParser extends HBaseSchemaParser { propertyNames.put("blockSizeKB", HBaseSchemaAttribute.BLOCKSIZE.name()); propertyNames.put("bloomFilter", HBaseSchemaAttribute.BLOOMFILTER.name()); propertyNames.put("inMemory", HBaseSchemaAttribute.IN_MEMORY.name()); + propertyNames.put("keepDeletedCells", HBaseSchemaAttribute.KEEP_DELETED_CELLS.name()); + propertyNames.put("minVersions", HBaseSchemaAttribute.MIN_VERSIONS.name()); propertyNames.put("maxVersions", HBaseSchemaAttribute.VERSIONS.name()); propertyNames.put("replicationScope", HBaseSchemaAttribute.REPLICATION_SCOPE.name()); propertyNames.put("timeToLiveMS", HBaseSchemaAttribute.TTL.name()); diff --git a/src/test/java/com/salesforce/scoot/ScootTest.java b/src/test/java/com/salesforce/scoot/ScootTest.java index e468fe2..b72eafd 100644 --- a/src/test/java/com/salesforce/scoot/ScootTest.java +++ b/src/test/java/com/salesforce/scoot/ScootTest.java @@ -27,10 +27,18 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; -import org.apache.hadoop.hbase.util.Bytes; +import java.net.URL; +import java.util.List; import junit.framework.TestCase; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.common.io.Resources; +import com.salesforce.scoot.parser.HBaseScootXMLParser; + /** * General tests for scoot functionality * @@ -65,4 +73,42 @@ public void testHelp() throws Exception { } } + + public void testParse() throws Exception { + URL xmlFile = Resources.getResource("ScootXMLParserTest.xml"); + String fileName = xmlFile.getFile(); + HBaseScootXMLParser parser = new HBaseScootXMLParser(); + parser.setResourceToParse(fileName); + HBaseSchema schema = parser.parse(); + + List tables = schema.getTables(); + assertEquals(1, tables.size()); + + // Test table attributes + HTableDescriptor table = tables.get(0); + assertEquals("testMe", table.getNameAsString()); + assertEquals("536870912", table.getValue(HBaseSchemaAttribute.MAX_FILESIZE.name())); + assertEquals("true", table.getValue(HBaseSchemaAttribute.READONLY.name())); + assertEquals("128974848", table.getValue(HBaseSchemaAttribute.MEMSTORE_FLUSHSIZE.name())); + assertEquals("true", table.getValue(HBaseSchemaAttribute.DEFERRED_LOG_FLUSH.name())); + assertEquals("ivarley", table.getOwnerString()); + + HColumnDescriptor[] columnFamilies = table.getColumnFamilies(); + assertNotNull(columnFamilies); + assertEquals(1, columnFamilies.length); + + // Test column family attributes + HColumnDescriptor columnFamily = columnFamilies[0]; + assertEquals("testMeColumnFamily1", columnFamily.getNameAsString()); + assertEquals("10", columnFamily.getValue(HBaseSchemaAttribute.VERSIONS.name())); + assertEquals("32768", columnFamily.getValue(HBaseSchemaAttribute.BLOCKSIZE.name())); + assertEquals("false", columnFamily.getValue(HBaseSchemaAttribute.BLOCKCACHE.name())); + assertEquals("123456789", columnFamily.getValue(HBaseSchemaAttribute.TTL.name())); + assertEquals("true", columnFamily.getValue(HBaseSchemaAttribute.IN_MEMORY.name())); + assertEquals("ROW", columnFamily.getValue(HBaseSchemaAttribute.BLOOMFILTER.name())); + assertEquals("true", columnFamily.getValue(HBaseSchemaAttribute.KEEP_DELETED_CELLS.name())); + assertEquals("5", columnFamily.getValue(HBaseSchemaAttribute.MIN_VERSIONS.name())); + assertEquals("1", columnFamily.getValue(HBaseSchemaAttribute.REPLICATION_SCOPE.name())); + + } } diff --git a/src/test/resources/ScootXMLParserTest.xml b/src/test/resources/ScootXMLParserTest.xml new file mode 100644 index 0000000..f62f46e --- /dev/null +++ b/src/test/resources/ScootXMLParserTest.xml @@ -0,0 +1,25 @@ + + + + + + + + +
+ + From 17d52893c7af499d60dfaa977fc8dbf5fb2ca5f5 Mon Sep 17 00:00:00 2001 From: Thomas Maurel Date: Wed, 21 Nov 2012 23:15:33 +0100 Subject: [PATCH 2/2] Fixed a bug in generated scripts pre-validation of column families attributes which caused warnings when not using default attributes : The HColumnDescriptor retrieved before each pre-validation was built from scratch and therefore holding default values of attributes. The column family is now retrieved using the table.getFamily(familyName) method. Tests have been updated to match this changes. --- .../scoot/scripter/HBaseRubySchemaPatchScripter.java | 3 ++- .../resources/DiffScriptGenerationTestResultAB.rb | 12 ++++++++---- .../resources/DiffScriptGenerationTestResultD.rb | 3 ++- .../resources/DiffScriptGenerationTestResultEF.rb | 6 ++++-- .../resources/DiffScriptGenerationTestResultFE.rb | 6 ++++-- .../resources/PhoenixScriptGenerationTestResultA.rb | 6 ++++-- 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/salesforce/scoot/scripter/HBaseRubySchemaPatchScripter.java b/src/main/java/com/salesforce/scoot/scripter/HBaseRubySchemaPatchScripter.java index 91d180f..44dfded 100644 --- a/src/main/java/com/salesforce/scoot/scripter/HBaseRubySchemaPatchScripter.java +++ b/src/main/java/com/salesforce/scoot/scripter/HBaseRubySchemaPatchScripter.java @@ -223,7 +223,8 @@ private void scriptVerifyTableMatches(HTableDescriptor oldTable, String operatio // now descend into child objects for (HColumnDescriptor c : oldTable.getColumnFamilies()){ s(" # Column family: " + c.getNameAsString()); - s(" cf = HColumnDescriptor.new(\"" + c.getNameAsString() + "\")"); + s(" cfname = \"" + c.getNameAsString() + "\""); + s(" cf = table.getFamily(cfname.bytes.to_a)"); for (Entry p : getSortedStringEntries(c.getValues())){ s(" compare(" + errorCollectionName + ", cf, \"" + operationName + "\", \"" + p.getKey() + "\", \"" + escapeDoubleQuotes(p.getValue()) + "\")"); diff --git a/src/test/resources/DiffScriptGenerationTestResultAB.rb b/src/test/resources/DiffScriptGenerationTestResultAB.rb index 5ec4689..612e730 100644 --- a/src/test/resources/DiffScriptGenerationTestResultAB.rb +++ b/src/test/resources/DiffScriptGenerationTestResultAB.rb @@ -74,7 +74,8 @@ def compare(errs, obj, action, attr, val) compare(preWarnings, table, "drop", "READONLY", "false") compare(preWarnings, table, "drop", "fullSchema", "
") # Column family: dropMeColumnFamily1 - cf = HColumnDescriptor.new("dropMeColumnFamily1") + cfname = "dropMeColumnFamily1" + cf = table.getFamily(cfname.bytes.to_a) compare(preWarnings, cf, "drop", "BLOCKCACHE", "true") compare(preWarnings, cf, "drop", "BLOCKSIZE", "65536") compare(preWarnings, cf, "drop", "BLOOMFILTER", "NONE") @@ -113,7 +114,8 @@ def compare(errs, obj, action, attr, val) compare(preErrors, table, "alter", "READONLY", "false") compare(preErrors, table, "alter", "fullSchema", "
") # Column family: alterMeColumnFamily1 - cf = HColumnDescriptor.new("alterMeColumnFamily1") + cfname = "alterMeColumnFamily1" + cf = table.getFamily(cfname.bytes.to_a) compare(preErrors, cf, "alter", "BLOCKCACHE", "true") compare(preErrors, cf, "alter", "BLOCKSIZE", "65536") compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE") @@ -257,7 +259,8 @@ def compare(errs, obj, action, attr, val) compare(preErrors, table, "create", "READONLY", "false") compare(preErrors, table, "create", "fullSchema", "
") # Column family: createMeColumnFamily1 - cf = HColumnDescriptor.new("createMeColumnFamily1") + cfname = "createMeColumnFamily1" + cf = table.getFamily(cfname.bytes.to_a) compare(preErrors, cf, "create", "BLOCKCACHE", "true") compare(preErrors, cf, "create", "BLOCKSIZE", "65536") compare(preErrors, cf, "create", "BLOOMFILTER", "NONE") @@ -290,7 +293,8 @@ def compare(errs, obj, action, attr, val) compare(preErrors, table, "alter", "READONLY", "false") compare(preErrors, table, "alter", "fullSchema", "
") # Column family: alterMeColumnFamily1 - cf = HColumnDescriptor.new("alterMeColumnFamily1") + cfname = "alterMeColumnFamily1" + cf = table.getFamily(cfname.bytes.to_a) compare(preErrors, cf, "alter", "BLOCKCACHE", "true") compare(preErrors, cf, "alter", "BLOCKSIZE", "66560") compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE") diff --git a/src/test/resources/DiffScriptGenerationTestResultD.rb b/src/test/resources/DiffScriptGenerationTestResultD.rb index 167d4fa..275cc20 100644 --- a/src/test/resources/DiffScriptGenerationTestResultD.rb +++ b/src/test/resources/DiffScriptGenerationTestResultD.rb @@ -66,7 +66,8 @@ def compare(errs, obj, action, attr, val) compare(preWarnings, table, "drop", "READONLY", "false") compare(preWarnings, table, "drop", "fullSchema", "
") # Column family: createMeColumnFamily1 - cf = HColumnDescriptor.new("createMeColumnFamily1") + cfname = "createMeColumnFamily1" + cf = table.getFamily(cfname.bytes.to_a) compare(preWarnings, cf, "drop", "BLOCKCACHE", "true") compare(preWarnings, cf, "drop", "BLOCKSIZE", "65536") compare(preWarnings, cf, "drop", "BLOOMFILTER", "NONE") diff --git a/src/test/resources/DiffScriptGenerationTestResultEF.rb b/src/test/resources/DiffScriptGenerationTestResultEF.rb index 9ed93be..3f8d5ce 100644 --- a/src/test/resources/DiffScriptGenerationTestResultEF.rb +++ b/src/test/resources/DiffScriptGenerationTestResultEF.rb @@ -66,7 +66,8 @@ def compare(errs, obj, action, attr, val) compare(preErrors, table, "alter", "READONLY", "false") compare(preErrors, table, "alter", "fullSchema", "
") # Column family: minimalColumnFamily1 - cf = HColumnDescriptor.new("minimalColumnFamily1") + cfname = "minimalColumnFamily1" + cf = table.getFamily(cfname.bytes.to_a) compare(preErrors, cf, "alter", "BLOCKCACHE", "true") compare(preErrors, cf, "alter", "BLOCKSIZE", "65536") compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE") @@ -159,7 +160,8 @@ def compare(errs, obj, action, attr, val) compare(preErrors, table, "alter", "READONLY", "false") compare(preErrors, table, "alter", "fullSchema", "
") # Column family: minimalColumnFamily1 - cf = HColumnDescriptor.new("minimalColumnFamily1") + cfname = "minimalColumnFamily1" + cf = table.getFamily(cfname.bytes.to_a) compare(preErrors, cf, "alter", "BLOCKCACHE", "true") compare(preErrors, cf, "alter", "BLOCKSIZE", "65536") compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE") diff --git a/src/test/resources/DiffScriptGenerationTestResultFE.rb b/src/test/resources/DiffScriptGenerationTestResultFE.rb index 63f10a7..7dabad3 100644 --- a/src/test/resources/DiffScriptGenerationTestResultFE.rb +++ b/src/test/resources/DiffScriptGenerationTestResultFE.rb @@ -66,7 +66,8 @@ def compare(errs, obj, action, attr, val) compare(preErrors, table, "alter", "READONLY", "false") compare(preErrors, table, "alter", "fullSchema", "
") # Column family: minimalColumnFamily1 - cf = HColumnDescriptor.new("minimalColumnFamily1") + cfname = "minimalColumnFamily1" + cf = table.getFamily(cfname.bytes.to_a) compare(preErrors, cf, "alter", "BLOCKCACHE", "true") compare(preErrors, cf, "alter", "BLOCKSIZE", "65536") compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE") @@ -159,7 +160,8 @@ def compare(errs, obj, action, attr, val) compare(preErrors, table, "alter", "READONLY", "false") compare(preErrors, table, "alter", "fullSchema", "
") # Column family: minimalColumnFamily1 - cf = HColumnDescriptor.new("minimalColumnFamily1") + cfname = "minimalColumnFamily1" + cf = table.getFamily(cfname.bytes.to_a) compare(preErrors, cf, "alter", "BLOCKCACHE", "true") compare(preErrors, cf, "alter", "BLOCKSIZE", "65536") compare(preErrors, cf, "alter", "BLOOMFILTER", "NONE") diff --git a/src/test/resources/PhoenixScriptGenerationTestResultA.rb b/src/test/resources/PhoenixScriptGenerationTestResultA.rb index feeefb0..45ed619 100644 --- a/src/test/resources/PhoenixScriptGenerationTestResultA.rb +++ b/src/test/resources/PhoenixScriptGenerationTestResultA.rb @@ -141,7 +141,8 @@ def compare(errs, obj, action, attr, val) compare(preErrors, table, "create", "MEMSTORE_FLUSHSIZE", "134217728") compare(preErrors, table, "create", "READONLY", "false") # Column family: 1 - cf = HColumnDescriptor.new("1") + cfname = "1" + cf = table.getFamily(cfname.bytes.to_a) compare(preErrors, cf, "create", "BLOCKCACHE", "true") compare(preErrors, cf, "create", "BLOCKSIZE", "65536") compare(preErrors, cf, "create", "BLOOMFILTER", "NONE") @@ -155,7 +156,8 @@ def compare(errs, obj, action, attr, val) compare(preErrors, cf, "create", "TTL", "2147483647") compare(preErrors, cf, "create", "VERSIONS", "3") # Column family: 2 - cf = HColumnDescriptor.new("2") + cfname = "2" + cf = table.getFamily(cfname.bytes.to_a) compare(preErrors, cf, "create", "BLOCKCACHE", "true") compare(preErrors, cf, "create", "BLOCKSIZE", "65536") compare(preErrors, cf, "create", "BLOOMFILTER", "NONE")