Skip to content

HBASE-24974: Provide a flexibility to print only row key and filter for multiple tables in the WALPrettyPrinter #2345

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bharathv merged 3 commits into from
Sep 9, 2020
Merged

HBASE-24974: Provide a flexibility to print only row key and filter for multiple tables in the WALPrettyPrinter #2345

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3ecf4e3
HBASE-24974: Provide a flexibility to print only row key and filter f…
Sep 3, 2020
f7af39e
Addressing comments
Sep 4, 2020
ecb8cf3
fixing check-s
Sep 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 45 additions & 17 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,14 @@
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
Expand All @@ -34,7 +37,6 @@
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
import org.apache.hadoop.hbase.util.Bytes;
Expand Down Expand Up @@ -79,9 +81,12 @@ public class WALPrettyPrinter {
private boolean outputJSON;
// The following enable filtering by sequence, region, and row, respectively
private long sequence;
private String table;

// List of tables for filter
private final Set<String> tableSet;
private String region;
private String row;
private boolean outputOnlyRowKey;
// enable in order to output a single list of transactions from several files
private boolean persistentOutput;
private boolean firstTxn;
Expand All @@ -99,9 +104,10 @@ public WALPrettyPrinter() {
outputValues = false;
outputJSON = false;
sequence = -1;
table = null;
tableSet = new HashSet<>();
region = null;
row = null;
outputOnlyRowKey = false;
persistentOutput = false;
firstTxn = true;
out = System.out;
Expand Down Expand Up @@ -147,11 +153,11 @@ public void setSequenceFilter(long sequence) {
}

/**
* Sets the table filter. Only log entries for this table are printed.
* @param table table name to set.
* Sets the tables filter. Only log entries for these tables are printed.
* @param tablesWithDelimiter table names separated with comma.
*/
public void setTableFilter(String table) {
this.table = table;
public void setTableFilter(String tablesWithDelimiter) {
Collections.addAll(tableSet, tablesWithDelimiter.split(","));
}
/**
* sets the region by which output will be filtered
Expand All @@ -165,7 +171,7 @@ public void setRegionFilter(String region) {
}

/**
* sets the region by which output will be filtered
* sets the row key by which output will be filtered
*
* @param row
* when not null, serves as a filter; only log entries from this row
Expand All @@ -175,6 +181,13 @@ public void setRowFilter(String row) {
this.row = row;
}

/**
* Option to print the row key only in case you just need the row keys from the WAL
*/
public void setOutputOnlyRowKey() {
this.outputOnlyRowKey = true;
}

/**
* sets the position to start seeking the WAL file
* @param position
Expand Down Expand Up @@ -274,7 +287,8 @@ public void processFile(final Configuration conf, final Path p)
Map<String, Object> txn = key.toStringMap();
long writeTime = key.getWriteTime();
// check output filters
if (table != null && !((TableName) txn.get("table")).toString().equals(table)) {
if (!tableSet.isEmpty() &&
Copy link
Contributor

@bharathv bharathv Sep 4, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I think the second condition covers the first one.

Copy link
Contributor Author

@sandeepvinayak sandeepvinayak Sep 4, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we need the first one where we don't pass -t param and set is Empty. In that case, the second one will always skip all the tables. @bharathv

!tableSet.contains(txn.get("table").toString())) {
continue;
}
if (sequence >= 0 && ((Long) txn.get("sequence")) != sequence) {
Expand All @@ -287,7 +301,7 @@ public void processFile(final Configuration conf, final Path p)
List<Map<String, Object>> actions = new ArrayList<>();
for (Cell cell : edit.getCells()) {
// add atomic operation to txn
Map<String, Object> op = new HashMap<>(toStringMap(cell));
Map<String, Object> op = new HashMap<>(toStringMap(cell, outputOnlyRowKey));
if (outputValues) {
op.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell)));
}
Expand Down Expand Up @@ -342,16 +356,20 @@ public static void printCell(PrintStream out, Map<String, Object> op, boolean ou
out.println("cell total size sum: " + op.get("total_size_sum"));
}

public static Map<String, Object> toStringMap(Cell cell) {
public static Map<String, Object> toStringMap(Cell cell, boolean printRowKeyOnly) {
Map<String, Object> stringMap = new HashMap<>();
stringMap.put("row",
Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));

if (printRowKeyOnly) {
return stringMap;
}
stringMap.put("type", cell.getType());
stringMap.put("family", Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
cell.getFamilyLength()));
cell.getFamilyLength()));
stringMap.put("qualifier",
Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
cell.getQualifierLength()));
Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
cell.getQualifierLength()));
stringMap.put("timestamp", cell.getTimestamp());
stringMap.put("vlen", cell.getValueLength());
if (cell.getTagsLength() > 0) {
Expand All @@ -360,13 +378,17 @@ public static Map<String, Object> toStringMap(Cell cell) {
while (tagsIterator.hasNext()) {
Tag tag = tagsIterator.next();
tagsString
.add((tag.getType()) + ":" + Bytes.toStringBinary(Tag.cloneValue(tag)));
.add((tag.getType()) + ":" + Bytes.toStringBinary(Tag.cloneValue(tag)));
}
stringMap.put("tag", tagsString);
}
return stringMap;
}

public static Map<String, Object> toStringMap(Cell cell) {
return toStringMap(cell, false);
}

public static void main(String[] args) throws IOException {
run(args);
}
Expand All @@ -386,11 +408,14 @@ public static void run(String[] args) throws IOException {
options.addOption("h", "help", false, "Output help message");
options.addOption("j", "json", false, "Output JSON");
options.addOption("p", "printvals", false, "Print values");
options.addOption("t", "table", true, "Table name to filter by.");
options.addOption("t", "tables", true,
"Table names (comma separated) to filter by; eg: test1,test2,test3 ");
options.addOption("r", "region", true,
"Region to filter by. Pass encoded region name; e.g. '9192caead6a5a20acb4454ffbc79fa14'");
options.addOption("s", "sequence", true,
"Sequence to filter by. Pass sequence number.");
options.addOption("k", "outputOnlyRowKey", false,
"Print only row keys");
options.addOption("w", "row", true, "Row to filter by. Pass row name.");
options.addOption("g", "goto", true, "Position to seek to in the file");

Expand All @@ -412,6 +437,9 @@ public static void run(String[] args) throws IOException {
if (cmd.hasOption("j")) {
printer.enableJSON();
}
if (cmd.hasOption("k")) {
printer.setOutputOnlyRowKey();
}
if (cmd.hasOption("t")) {
printer.setTableFilter(cmd.getOptionValue("t"));
}
Expand Down