Skip to content

Commit

Permalink
HBASE-18503 Change ***Util and Master to use TableDescriptor and Colu…
Browse files Browse the repository at this point in the history 
…mnFamilyDescriptor
  • Loading branch information
Chia-Ping Tsai committed Aug 24, 2017
1 parent ae3b51a commit b033486
Show file tree
Hide file tree
Showing 77 changed files with 942 additions and 977 deletions.
4 changes: 2 additions & 2 deletions hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java
View file
Expand Up @@ -44,7 +44,6 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
Expand All @@ -56,6 +55,7 @@
import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
Expand Down Expand Up @@ -139,7 +139,7 @@ public static HashMap<String, Long> getRSLogTimestampMins(
LOG.warn("Table " + table + " does not exists, skipping it.");
continue;
}
HTableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, table);
TableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, table);

// write a copy of descriptor to the target directory
Path target = new Path(backupInfo.getTableBackupDir(table));
Expand Down
48 changes: 25 additions & 23 deletions hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java
View file
Expand Up @@ -33,16 +33,17 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.BackupRestoreFactory;
import org.apache.hadoop.hbase.backup.HBackupFileSystem;
import org.apache.hadoop.hbase.backup.RestoreJob;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
Expand Down Expand Up @@ -122,10 +123,10 @@ ArrayList<Path> getRegionList(TableName tableName) throws FileNotFoundException,
}


void modifyTableSync(Connection conn, HTableDescriptor desc) throws IOException {
void modifyTableSync(Connection conn, TableDescriptor desc) throws IOException {

try (Admin admin = conn.getAdmin();) {
admin.modifyTable(desc.getTableName(), desc);
admin.modifyTable(desc);
int attempt = 0;
int maxAttempts = 600;
while (!admin.isTableAvailable(desc.getTableName())) {
Expand Down Expand Up @@ -172,29 +173,30 @@ public void incrementalRestoreTable(Connection conn, Path tableBackupPath, Path[
// adjust table schema
for (int i = 0; i < tableNames.length; i++) {
TableName tableName = tableNames[i];
HTableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, incrBackupId);
TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, incrBackupId);
LOG.debug("Found descriptor " + tableDescriptor + " through " + incrBackupId);

TableName newTableName = newTableNames[i];
HTableDescriptor newTableDescriptor = new HTableDescriptor(admin.getTableDescriptor(newTableName));
List<HColumnDescriptor> families = Arrays.asList(tableDescriptor.getColumnFamilies());
List<HColumnDescriptor> existingFamilies =
TableDescriptor newTableDescriptor = admin.listTableDescriptor(newTableName);
List<ColumnFamilyDescriptor> families = Arrays.asList(tableDescriptor.getColumnFamilies());
List<ColumnFamilyDescriptor> existingFamilies =
Arrays.asList(newTableDescriptor.getColumnFamilies());
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(newTableDescriptor);
boolean schemaChangeNeeded = false;
for (HColumnDescriptor family : families) {
for (ColumnFamilyDescriptor family : families) {
if (!existingFamilies.contains(family)) {
newTableDescriptor.addFamily(family);
builder.addColumnFamily(family);
schemaChangeNeeded = true;
}
}
for (HColumnDescriptor family : existingFamilies) {
for (ColumnFamilyDescriptor family : existingFamilies) {
if (!families.contains(family)) {
newTableDescriptor.removeFamily(family.getName());
builder.removeColumnFamily(family.getName());
schemaChangeNeeded = true;
}
}
if (schemaChangeNeeded) {
modifyTableSync(conn, newTableDescriptor);
modifyTableSync(conn, builder.build());
LOG.info("Changed " + newTableDescriptor.getTableName() + " to: " + newTableDescriptor);
}
}
Expand Down Expand Up @@ -253,24 +255,24 @@ Path getTableInfoPath(TableName tableName) throws FileNotFoundException, IOExcep
/**
* Get table descriptor
* @param tableName is the table backed up
* @return {@link HTableDescriptor} saved in backup image of the table
* @return {@link TableDescriptor} saved in backup image of the table
*/
HTableDescriptor getTableDesc(TableName tableName) throws FileNotFoundException, IOException {
TableDescriptor getTableDesc(TableName tableName) throws FileNotFoundException, IOException {
Path tableInfoPath = this.getTableInfoPath(tableName);
SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
HTableDescriptor tableDescriptor = manifest.getTableDescriptor();
TableDescriptor tableDescriptor = manifest.getTableDescriptor();
if (!tableDescriptor.getTableName().equals(tableName)) {
LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
+ tableInfoPath.toString());
LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getNameAsString());
LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getTableName().getNameAsString());
throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName
+ " under tableInfoPath: " + tableInfoPath.toString());
}
return tableDescriptor;
}

private HTableDescriptor getTableDescriptor(FileSystem fileSys, TableName tableName,
private TableDescriptor getTableDescriptor(FileSystem fileSys, TableName tableName,
String lastIncrBackupId) throws IOException {
if (lastIncrBackupId != null) {
String target =
Expand All @@ -289,7 +291,7 @@ private void createAndRestoreTable(Connection conn, TableName tableName, TableNa
FileSystem fileSys = tableBackupPath.getFileSystem(this.conf);

// get table descriptor first
HTableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, lastIncrBackupId);
TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, lastIncrBackupId);
if (tableDescriptor != null) {
LOG.debug("Retrieved descriptor: " + tableDescriptor + " thru " + lastIncrBackupId);
}
Expand Down Expand Up @@ -325,7 +327,7 @@ private void createAndRestoreTable(Connection conn, TableName tableName, TableNa
LOG.debug("find table descriptor but no archive dir for table " + tableName
+ ", will only create table");
}
tableDescriptor = new HTableDescriptor(newTableName, tableDescriptor);
tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, null, tableDescriptor,
truncateIfExists);
return;
Expand All @@ -336,9 +338,9 @@ private void createAndRestoreTable(Connection conn, TableName tableName, TableNa
}

if (tableDescriptor == null) {
tableDescriptor = new HTableDescriptor(newTableName);
tableDescriptor = TableDescriptorBuilder.newBuilder(newTableName).build();
} else {
tableDescriptor = new HTableDescriptor(newTableName, tableDescriptor);
tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
}

// record all region dirs:
Expand Down Expand Up @@ -470,7 +472,7 @@ byte[][] generateBoundaryKeys(ArrayList<Path> regionDirList) throws FileNotFound
* @throws IOException exception
*/
private void checkAndCreateTable(Connection conn, Path tableBackupPath, TableName tableName,
TableName targetTableName, ArrayList<Path> regionDirList, HTableDescriptor htd,
TableName targetTableName, ArrayList<Path> regionDirList, TableDescriptor htd,
boolean truncateIfExists) throws IOException {
try (Admin admin = conn.getAdmin();) {
boolean createNew = false;
Expand Down
11 changes: 4 additions & 7 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
View file
Expand Up @@ -639,13 +639,10 @@ public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
if (obj instanceof HColumnDescriptor) {
return delegatee.equals(((HColumnDescriptor) obj).delegatee);
}
if (!(obj instanceof HColumnDescriptor)) {
return false;
}
return compareTo((HColumnDescriptor)obj) == 0;
return false;
}

/**
Expand All @@ -658,7 +655,7 @@ public int hashCode() {

@Override
public int compareTo(HColumnDescriptor other) {
return delegatee.compareTo(other.delegatee);
return COMPARATOR.compare(this, other);
}

/**
Expand Down
11 changes: 4 additions & 7 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
View file
Expand Up @@ -495,13 +495,10 @@ public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
if (obj instanceof HTableDescriptor) {
return delegatee.equals(((HTableDescriptor) obj).delegatee);
}
if (!(obj instanceof HTableDescriptor)) {
return false;
}
return compareTo((HTableDescriptor)obj) == 0;
return false;
}

/**
Expand All @@ -523,7 +520,7 @@ public int hashCode() {
*/
@Override
public int compareTo(final HTableDescriptor other) {
return delegatee.compareTo(other.delegatee);
return TableDescriptor.COMPARATOR.compare(this, other);
}

/**
Expand Down
13 changes: 5 additions & 8 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java
View file
Expand Up @@ -1160,13 +1160,10 @@ public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
if (obj instanceof ModifyableColumnFamilyDescriptor) {
return ColumnFamilyDescriptor.COMPARATOR.compare(this, (ModifyableColumnFamilyDescriptor) obj) == 0;
}
if (!(obj instanceof ModifyableColumnFamilyDescriptor)) {
return false;
}
return compareTo((ModifyableColumnFamilyDescriptor) obj) == 0;
return false;
}

@Override
Expand All @@ -1188,7 +1185,7 @@ public int compareTo(ModifyableColumnFamilyDescriptor other) {
* @see #parseFrom(byte[])
*/
private byte[] toByteArray() {
return ProtobufUtil.prependPBMagic(ProtobufUtil.convertToColumnFamilySchema(this)
return ProtobufUtil.prependPBMagic(ProtobufUtil.toColumnFamilySchema(this)
.toByteArray());
}

Expand All @@ -1213,7 +1210,7 @@ private static ColumnFamilyDescriptor parseFrom(final byte[] bytes) throws Deser
} catch (IOException e) {
throw new DeserializationException(e);
}
return ProtobufUtil.convertToColumnDesc(cfs);
return ProtobufUtil.toColumnFamilyDescriptor(cfs);
}

@Override
Expand Down
41 changes: 11 additions & 30 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
View file
Expand Up @@ -378,7 +378,7 @@ protected List<TableDescriptor> rpcCall() throws Exception {
.setNamespaceName(Bytes.toString(name)).build())
.getTableSchemaList()
.stream()
.map(ProtobufUtil::convertToTableDesc)
.map(ProtobufUtil::toTableDescriptor)
.collect(Collectors.toList());
}
});
Expand Down Expand Up @@ -459,8 +459,8 @@ public HTableDescriptor[] listTables(final Pattern pattern, final boolean includ
protected HTableDescriptor[] rpcCall() throws Exception {
GetTableDescriptorsRequest req =
RequestConverter.buildGetTableDescriptorsRequest(pattern, includeSysTables);
return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(getRpcController(),
req));
return ProtobufUtil.toTableDescriptorList(master.getTableDescriptors(getRpcController(),
req)).stream().map(ImmutableHTableDescriptor::new).toArray(HTableDescriptor[]::new);
}
});
}
Expand Down Expand Up @@ -525,7 +525,7 @@ protected TableDescriptor rpcCall() throws Exception {
RequestConverter.buildGetTableDescriptorsRequest(tableName);
GetTableDescriptorsResponse htds = master.getTableDescriptors(getRpcController(), req);
if (!htds.getTableSchemaList().isEmpty()) {
return ProtobufUtil.convertToTableDesc(htds.getTableSchemaList().get(0));
return ProtobufUtil.toTableDescriptor(htds.getTableSchemaList().get(0));
}
return null;
}
Expand Down Expand Up @@ -554,7 +554,7 @@ protected HTableDescriptor rpcCall() throws Exception {
RequestConverter.buildGetTableDescriptorsRequest(tableName);
GetTableDescriptorsResponse htds = master.getTableDescriptors(getRpcController(), req);
if (!htds.getTableSchemaList().isEmpty()) {
return ProtobufUtil.convertToHTableDesc(htds.getTableSchemaList().get(0));
return new ImmutableHTableDescriptor(ProtobufUtil.toTableDescriptor(htds.getTableSchemaList().get(0)));
}
return null;
}
Expand Down Expand Up @@ -2300,7 +2300,7 @@ protected HTableDescriptor[] rpcCall() throws Exception {
.build()).getTableSchemaList();
HTableDescriptor[] res = new HTableDescriptor[list.size()];
for(int i=0; i < list.size(); i++) {
res[i] = new ImmutableHTableDescriptor(ProtobufUtil.convertToHTableDesc(list.get(i)));
res[i] = new ImmutableHTableDescriptor(ProtobufUtil.toTableDescriptor(list.get(i)));
}
return res;
}
Expand Down Expand Up @@ -2419,33 +2419,14 @@ public HTableDescriptor[] getTableDescriptorsByTableName(final List<TableName> t
protected HTableDescriptor[] rpcCall() throws Exception {
GetTableDescriptorsRequest req =
RequestConverter.buildGetTableDescriptorsRequest(tableNames);
return ProtobufUtil.
getHTableDescriptorArray(master.getTableDescriptors(getRpcController(), req));
return ProtobufUtil.toTableDescriptorList(master.getTableDescriptors(getRpcController(), req))
.stream()
.map(ImmutableHTableDescriptor::new)
.toArray(HTableDescriptor[]::new);
}
});
}

/**
* Get tableDescriptor
* @param tableName one table name
* @return HTD the HTableDescriptor or null if the table not exists
* @throws IOException if a remote or network exception occurs
*/
private HTableDescriptor getTableDescriptorByTableName(TableName tableName)
throws IOException {
List<TableName> tableNames = new ArrayList<>(1);
tableNames.add(tableName);

HTableDescriptor[] htdl = getTableDescriptorsByTableName(tableNames);

if (htdl == null || htdl.length == 0) {
return null;
}
else {
return htdl[0];
}
}

@Override
public HTableDescriptor[] getTableDescriptors(List<String> names)
throws IOException {
Expand Down Expand Up @@ -3709,7 +3690,7 @@ protected TableName getTableName() {
* @return the table descriptor
*/
protected TableDescriptor getTableDescriptor() throws IOException {
return getAdmin().getTableDescriptorByTableName(getTableName());
return getAdmin().listTableDescriptor(getTableName());
}

/**
Expand Down
2 changes: 1 addition & 1 deletion hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
View file
Expand Up @@ -453,7 +453,7 @@ this.<List<TableSchema>> newMasterCaller()
return;
}
if (!tableSchemas.isEmpty()) {
future.complete(ProtobufUtil.convertToTableDesc(tableSchemas.get(0)));
future.complete(ProtobufUtil.toTableDescriptor(tableSchemas.get(0)));
} else {
future.completeExceptionally(new TableNotFoundException(tableName.getNameAsString()));
}
Expand Down

0 comments on commit b033486

Please sign in to comment.