From 3ecf4e3f5c78d95a9057c76902ac9c1c742364f5 Mon Sep 17 00:00:00 2001 From: Sandeep Pal Date: Wed, 2 Sep 2020 22:01:01 -0700 Subject: [PATCH 1/3] HBASE-24974: Provide a flexibility to print only row key and filter for multiple tables in the WALPrettyPrinter --- .../hadoop/hbase/wal/WALPrettyPrinter.java | 59 ++++++++++++++----- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java index a9d089877a30..78a39c7ce9ca 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java @@ -23,9 +23,11 @@ import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -79,9 +81,12 @@ public class WALPrettyPrinter { private boolean outputJSON; // The following enable filtering by sequence, region, and row, respectively private long sequence; - private String table; + + // List of tables for filter + private Set tableList; private String region; private String row; + private boolean outputOnlyRowKey; // enable in order to output a single list of transactions from several files private boolean persistentOutput; private boolean firstTxn; @@ -99,9 +104,10 @@ public WALPrettyPrinter() { outputValues = false; outputJSON = false; sequence = -1; - table = null; + tableList = new HashSet<>(); region = null; row = null; + outputOnlyRowKey = false; persistentOutput = false; firstTxn = true; out = System.out; @@ -147,11 +153,13 @@ public void setSequenceFilter(long sequence) { } /** - * Sets the table filter. Only log entries for this table are printed. - * @param table table name to set. + * Sets the tables filter. Only log entries for these table are printed. + * @param tableListWithDelimiter table names separated with comma. */ - public void setTableFilter(String table) { - this.table = table; + public void setTableFilter(String tableListWithDelimiter) { + for (String table : tableListWithDelimiter.split(",")) { + tableList.add(table); + } } /** * sets the region by which output will be filtered @@ -165,7 +173,7 @@ public void setRegionFilter(String region) { } /** - * sets the region by which output will be filtered + * sets the row key by which output will be filtered * * @param row * when not null, serves as a filter; only log entries from this row @@ -175,6 +183,13 @@ public void setRowFilter(String row) { this.row = row; } + /** + * Option to print the row key only in case you just need the row keys from the WAL + */ + public void setOutputOnlyRowKey() { + this.outputOnlyRowKey = true; + } + /** * sets the position to start seeking the WAL file * @param position @@ -274,7 +289,8 @@ public void processFile(final Configuration conf, final Path p) Map txn = key.toStringMap(); long writeTime = key.getWriteTime(); // check output filters - if (table != null && !((TableName) txn.get("table")).toString().equals(table)) { + if (!tableList.isEmpty() && + !tableList.contains(((TableName) txn.get("table")).toString())) { continue; } if (sequence >= 0 && ((Long) txn.get("sequence")) != sequence) { @@ -287,7 +303,7 @@ public void processFile(final Configuration conf, final Path p) List> actions = new ArrayList<>(); for (Cell cell : edit.getCells()) { // add atomic operation to txn - Map op = new HashMap<>(toStringMap(cell)); + Map op = new HashMap<>(toStringMap(cell, outputOnlyRowKey)); if (outputValues) { op.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell))); } @@ -342,16 +358,20 @@ public static void printCell(PrintStream out, Map op, boolean ou out.println("cell total size sum: " + op.get("total_size_sum")); } - public static Map toStringMap(Cell cell) { + public static Map toStringMap(Cell cell, boolean printRowKeyOnly) { Map stringMap = new HashMap<>(); stringMap.put("row", - Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); + Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); + + if (printRowKeyOnly) { + return stringMap; + } stringMap.put("type", cell.getType()); stringMap.put("family", Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(), - cell.getFamilyLength())); + cell.getFamilyLength())); stringMap.put("qualifier", - Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength())); + Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(), + cell.getQualifierLength())); stringMap.put("timestamp", cell.getTimestamp()); stringMap.put("vlen", cell.getValueLength()); if (cell.getTagsLength() > 0) { @@ -360,13 +380,17 @@ public static Map toStringMap(Cell cell) { while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); tagsString - .add((tag.getType()) + ":" + Bytes.toStringBinary(Tag.cloneValue(tag))); + .add((tag.getType()) + ":" + Bytes.toStringBinary(Tag.cloneValue(tag))); } stringMap.put("tag", tagsString); } return stringMap; } + public static Map toStringMap(Cell cell) { + return toStringMap(cell, false); + } + public static void main(String[] args) throws IOException { run(args); } @@ -391,6 +415,8 @@ public static void run(String[] args) throws IOException { "Region to filter by. Pass encoded region name; e.g. '9192caead6a5a20acb4454ffbc79fa14'"); options.addOption("s", "sequence", true, "Sequence to filter by. Pass sequence number."); + options.addOption("k", "outputOnlyRowKey", false, + "If only row keys are required in output"); options.addOption("w", "row", true, "Row to filter by. Pass row name."); options.addOption("g", "goto", true, "Position to seek to in the file"); @@ -412,6 +438,9 @@ public static void run(String[] args) throws IOException { if (cmd.hasOption("j")) { printer.enableJSON(); } + if (cmd.hasOption("k")) { + printer.setOutputOnlyRowKey(); + } if (cmd.hasOption("t")) { printer.setTableFilter(cmd.getOptionValue("t")); } From f7af39eaf5909e2307c8cf0637da051f809b24bf Mon Sep 17 00:00:00 2001 From: Sandeep Pal Date: Thu, 3 Sep 2020 20:44:40 -0700 Subject: [PATCH 2/3] Addressing comments --- .../hadoop/hbase/wal/WALPrettyPrinter.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java index 78a39c7ce9ca..43703b34fff9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -83,7 +84,7 @@ public class WALPrettyPrinter { private long sequence; // List of tables for filter - private Set tableList; + private final Set tableSet; private String region; private String row; private boolean outputOnlyRowKey; @@ -104,7 +105,7 @@ public WALPrettyPrinter() { outputValues = false; outputJSON = false; sequence = -1; - tableList = new HashSet<>(); + tableSet = new HashSet<>(); region = null; row = null; outputOnlyRowKey = false; @@ -153,13 +154,11 @@ public void setSequenceFilter(long sequence) { } /** - * Sets the tables filter. Only log entries for these table are printed. - * @param tableListWithDelimiter table names separated with comma. + * Sets the tables filter. Only log entries for these tables are printed. + * @param tablesWithDelimiter table names separated with comma. */ - public void setTableFilter(String tableListWithDelimiter) { - for (String table : tableListWithDelimiter.split(",")) { - tableList.add(table); - } + public void setTableFilter(String tablesWithDelimiter) { + Collections.addAll(tableSet, tablesWithDelimiter.split(",")); } /** * sets the region by which output will be filtered @@ -289,8 +288,8 @@ public void processFile(final Configuration conf, final Path p) Map txn = key.toStringMap(); long writeTime = key.getWriteTime(); // check output filters - if (!tableList.isEmpty() && - !tableList.contains(((TableName) txn.get("table")).toString())) { + if (!tableSet.isEmpty() && + !tableSet.contains(txn.get("table").toString())) { continue; } if (sequence >= 0 && ((Long) txn.get("sequence")) != sequence) { @@ -410,13 +409,14 @@ public static void run(String[] args) throws IOException { options.addOption("h", "help", false, "Output help message"); options.addOption("j", "json", false, "Output JSON"); options.addOption("p", "printvals", false, "Print values"); - options.addOption("t", "table", true, "Table name to filter by."); + options.addOption("t", "tables", true, + "Table names (comma separated) to filter by; eg: test1,test2,test3 "); options.addOption("r", "region", true, "Region to filter by. Pass encoded region name; e.g. '9192caead6a5a20acb4454ffbc79fa14'"); options.addOption("s", "sequence", true, "Sequence to filter by. Pass sequence number."); options.addOption("k", "outputOnlyRowKey", false, - "If only row keys are required in output"); + "Print only row keys"); options.addOption("w", "row", true, "Row to filter by. Pass row name."); options.addOption("g", "goto", true, "Position to seek to in the file"); From ecb8cf32756553ba0b2b467f2d65efd1c4129ecc Mon Sep 17 00:00:00 2001 From: Sandeep Pal Date: Fri, 4 Sep 2020 00:03:02 -0700 Subject: [PATCH 3/3] fixing check-s --- .../main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java index 43703b34fff9..a37efec610eb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.PrivateCellUtil; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader; import org.apache.hadoop.hbase.util.Bytes;