diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateDatabaseHook.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateDatabaseHook.java index 4d2f3d82116d..a1c1f24f38b0 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateDatabaseHook.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateDatabaseHook.java @@ -90,6 +90,7 @@ protected void authorizeDDLWork(HiveSemanticAnalyzerHookContext context, if (createDb != null) { Database db = new Database(createDb.getName(), createDb.getComment(), createDb.getLocationUri(), createDb.getDatabaseProperties()); + db.setExternalLocationUri(createDb.getLocationExternalUri()); authorize(db, Privilege.CREATE); } } diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateDBDesc.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateDBDesc.java index f2373a4aac1d..0bf20297f680 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateDBDesc.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateDBDesc.java @@ -34,6 +34,7 @@ public class HCatCreateDBDesc { private String dbName; private String locationUri; + private String externalLocationUri; private String comment; private Map dbProperties; private boolean ifNotExits = false; @@ -74,6 +75,15 @@ public String getLocation() { return this.locationUri; } + /** + * Gets the external location. + * + * @return the external location + */ + public String getExternalLocation() { + return this.externalLocationUri; + } + /** * Gets the database name. * @@ -93,6 +103,8 @@ public String toString() { + (dbName != null ? "dbName=" + dbName + ", " : "dbName=null") + (locationUri != null ? "location=" + locationUri + ", " : "location=null") + + (externalLocationUri != null ? "externalLocationUri=" + externalLocationUri + ", " + : "externalLocationUri=null") + (comment != null ? "comment=" + comment + ", " : "comment=null") + (dbProperties != null ? "dbProperties=" + dbProperties + ", " : "dbProperties=null") + "ifNotExits=" + ifNotExits + "]"; @@ -112,6 +124,7 @@ Database toHiveDb() { Database hiveDB = new Database(); hiveDB.setDescription(this.comment); hiveDB.setLocationUri(this.locationUri); + hiveDB.setExternalLocationUri(this.externalLocationUri); hiveDB.setName(this.dbName); hiveDB.setParameters(this.dbProperties); return hiveDB; @@ -120,6 +133,7 @@ Database toHiveDb() { public static class Builder { private String innerLoc; + private String innerExternalLoc; private String innerComment; private Map innerDBProps; private String dbName; @@ -140,6 +154,17 @@ public Builder location(String value) { return this; } + /** + * External location. + * + * @param value the external location of the database. + * @return the builder + */ + public Builder externalLocation(String value) { + this.innerExternalLoc = value; + return this; + } + /** * Comment. * @@ -187,6 +212,7 @@ public HCatCreateDBDesc build() throws HCatException { HCatCreateDBDesc desc = new HCatCreateDBDesc(this.dbName); desc.comment = this.innerComment; desc.locationUri = this.innerLoc; + desc.externalLocationUri = this.innerExternalLoc; desc.dbProperties = this.innerDBProps; desc.ifNotExits = this.ifNotExists; return desc; diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatDatabase.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatDatabase.java index a13572df4ef5..470a8d6f0bae 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatDatabase.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatDatabase.java @@ -34,6 +34,7 @@ public class HCatDatabase { private String dbName; private String dbLocation; + private String dbExternalLocation; private String comment; private Map props; @@ -41,18 +42,28 @@ public class HCatDatabase { this.dbName = db.getName(); this.props = db.getParameters(); this.dbLocation = db.getLocationUri(); + this.dbExternalLocation = db.getExternalLocationUri(); this.comment = db.getDescription(); } /** * Gets the database name. - * + *public String getExternalLocation() { * @return the database name */ public String getName() { return dbName; } + /** + * Gets the external dB location. + * + * @return the external dB location + */ + public String getExternalLocation() { + return dbExternalLocation; + } + /** * Gets the dB location. * @@ -85,6 +96,7 @@ public String toString() { return "HCatDatabase [" + (dbName != null ? "dbName=" + dbName + ", " : "dbName=null") + (dbLocation != null ? "dbLocation=" + dbLocation + ", " : "dbLocation=null") + + (dbExternalLocation != null ? "dbExternalLocation=" + dbExternalLocation + ", " : "dbExternalLocation=null") + (comment != null ? "comment=" + comment + ", " : "comment=null") + (props != null ? "props=" + props : "props=null") + "]"; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index b7babd623d78..89087bfc3c16 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4824,6 +4824,7 @@ private int createDatabase(Hive db, CreateDatabaseDesc crtDb) database.setName(crtDb.getName()); database.setDescription(crtDb.getComment()); database.setLocationUri(crtDb.getLocationUri()); + database.setExternalLocationUri(crtDb.getLocationExternalUri()); database.setParameters(crtDb.getDatabaseProperties()); database.setOwnerName(SessionState.getUserFromAuthenticator()); database.setOwnerType(PrincipalType.USER); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateDatabaseDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateDatabaseDesc.java index f2e6a7708390..0dd84ee1709b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateDatabaseDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateDatabaseDesc.java @@ -34,6 +34,7 @@ public class CreateDatabaseDesc extends DDLDesc implements Serializable { String databaseName; String locationUri; + String locationExternalUri; String comment; boolean ifNotExists; Map dbProperties; @@ -97,4 +98,13 @@ public String getLocationUri() { public void setLocationUri(String locationUri) { this.locationUri = locationUri; } + + @Explain(displayName="externalLocationUri") + public String getLocationExternalUri() { + return locationExternalUri; + } + + public void setLocationExternalUri(String locationExternalUri) { + this.locationExternalUri = locationExternalUri; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java index de5504498dea..6a5ab6ffbeb6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.util.Optional; import javax.security.auth.login.LoginException; @@ -138,7 +139,12 @@ public void authorize(Privilege[] readRequiredPriv, Privilege[] writeRequiredPri @Override public void authorize(Database db, Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) throws HiveException, AuthorizationException { - Path path = getDbLocation(db); + Path path = getExternalDbLocation(db); + + if (path == null) { + throw new AuthorizationException("Couldn't determine external directory for database " + + db.getName() + ", which the authorization is based on"); + } // extract drop privileges DropPrivilegeExtractor privExtractor = new DropPrivilegeExtractor(readRequiredPriv, @@ -430,6 +436,25 @@ protected Path getDbLocation(Database db) throws HiveException { } } + private Path getExternalDbLocation(Database db) throws HiveException { + try { + initWh(); + String location = db.getExternalLocationUri(); + if (location == null) { + return wh.getDefaultExternalDatabasePath(db.getName()); + } else { + Optional path = wh.getDatabaseExternalPath(db); + if (path.isPresent()) { + return wh.getDnsPath(path.get()); + } else { + return null; + } + } + } catch (MetaException ex) { + throw hiveException(ex); + } + } + private HiveException hiveException(Exception e) { return new HiveException(e); } diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 67d8fb41d196..baa6b3db4c5a 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -45,6 +45,7 @@ ${project.basedir}/src/test/resources ${project.build.directory}/tmp ${project.build.directory}/warehouse + ${project.build.directory}/external file:// 1 true @@ -580,8 +581,10 @@ + + @@ -745,6 +748,7 @@ false ${test.tmp.dir} ${test.warehouse.scheme}${test.warehouse.dir} + ${test.warehouse.scheme}${test.warehouse.external.dir} ${log4j.conf.dir} diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp index bc4d168a74d5..c75741ae5e6c 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp +++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp @@ -4456,6 +4456,11 @@ void Catalog::__set_locationUri(const std::string& val) { this->locationUri = val; } +void Catalog::__set_externalLocationUri(const std::string& val) { + this->externalLocationUri = val; +__isset.externalLocationUri = true; +} + uint32_t Catalog::read(::apache::thrift::protocol::TProtocol* iprot) { apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); @@ -4501,6 +4506,14 @@ uint32_t Catalog::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; + case 4: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->externalLocationUri); + this->__isset.externalLocationUri = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -4531,6 +4544,11 @@ uint32_t Catalog::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeString(this->locationUri); xfer += oprot->writeFieldEnd(); + if (this->__isset.externalLocationUri) { + xfer += oprot->writeFieldBegin("externalLocationUri", ::apache::thrift::protocol::T_STRING, 4); + xfer += oprot->writeString(this->externalLocationUri); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -4541,6 +4559,7 @@ void swap(Catalog &a, Catalog &b) { swap(a.name, b.name); swap(a.description, b.description); swap(a.locationUri, b.locationUri); + swap(a.externalLocationUri, b.externalLocationUri); swap(a.__isset, b.__isset); } @@ -4548,12 +4567,14 @@ Catalog::Catalog(const Catalog& other130) { name = other130.name; description = other130.description; locationUri = other130.locationUri; + externalLocationUri = other130.externalLocationUri; __isset = other130.__isset; } Catalog& Catalog::operator=(const Catalog& other131) { name = other131.name; description = other131.description; locationUri = other131.locationUri; + externalLocationUri = other131.externalLocationUri; __isset = other131.__isset; return *this; } @@ -4563,6 +4584,7 @@ void Catalog::printTo(std::ostream& out) const { out << "name=" << to_string(name); out << ", " << "description="; (__isset.description ? (out << to_string(description)) : (out << "")); out << ", " << "locationUri=" << to_string(locationUri); + out << ", " << "externalLocationUri="; (__isset.externalLocationUri ? (out << to_string(externalLocationUri)) : (out << "")); out << ")"; } @@ -5139,6 +5161,11 @@ void Database::__set_locationUri(const std::string& val) { this->locationUri = val; } +void Database::__set_externalLocationUri(const std::string& val) { + this->externalLocationUri = val; +__isset.externalLocationUri = true; +} + void Database::__set_parameters(const std::map & val) { this->parameters = val; } @@ -5209,6 +5236,14 @@ uint32_t Database::read(::apache::thrift::protocol::TProtocol* iprot) { } break; case 4: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->externalLocationUri); + this->__isset.externalLocationUri = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: if (ftype == ::apache::thrift::protocol::T_MAP) { { this->parameters.clear(); @@ -5231,7 +5266,7 @@ uint32_t Database::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; - case 5: + case 6: if (ftype == ::apache::thrift::protocol::T_STRUCT) { xfer += this->privileges.read(iprot); this->__isset.privileges = true; @@ -5239,7 +5274,7 @@ uint32_t Database::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; - case 6: + case 7: if (ftype == ::apache::thrift::protocol::T_STRING) { xfer += iprot->readString(this->ownerName); this->__isset.ownerName = true; @@ -5247,7 +5282,7 @@ uint32_t Database::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; - case 7: + case 8: if (ftype == ::apache::thrift::protocol::T_I32) { int32_t ecast157; xfer += iprot->readI32(ecast157); @@ -5257,7 +5292,7 @@ uint32_t Database::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; - case 8: + case 9: if (ftype == ::apache::thrift::protocol::T_STRING) { xfer += iprot->readString(this->catalogName); this->__isset.catalogName = true; @@ -5294,7 +5329,12 @@ uint32_t Database::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeString(this->locationUri); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("parameters", ::apache::thrift::protocol::T_MAP, 4); + if (this->__isset.externalLocationUri) { + xfer += oprot->writeFieldBegin("externalLocationUri", ::apache::thrift::protocol::T_STRING, 4); + xfer += oprot->writeString(this->externalLocationUri); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldBegin("parameters", ::apache::thrift::protocol::T_MAP, 5); { xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->parameters.size())); std::map ::const_iterator _iter158; @@ -5308,22 +5348,22 @@ uint32_t Database::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldEnd(); if (this->__isset.privileges) { - xfer += oprot->writeFieldBegin("privileges", ::apache::thrift::protocol::T_STRUCT, 5); + xfer += oprot->writeFieldBegin("privileges", ::apache::thrift::protocol::T_STRUCT, 6); xfer += this->privileges.write(oprot); xfer += oprot->writeFieldEnd(); } if (this->__isset.ownerName) { - xfer += oprot->writeFieldBegin("ownerName", ::apache::thrift::protocol::T_STRING, 6); + xfer += oprot->writeFieldBegin("ownerName", ::apache::thrift::protocol::T_STRING, 7); xfer += oprot->writeString(this->ownerName); xfer += oprot->writeFieldEnd(); } if (this->__isset.ownerType) { - xfer += oprot->writeFieldBegin("ownerType", ::apache::thrift::protocol::T_I32, 7); + xfer += oprot->writeFieldBegin("ownerType", ::apache::thrift::protocol::T_I32, 8); xfer += oprot->writeI32((int32_t)this->ownerType); xfer += oprot->writeFieldEnd(); } if (this->__isset.catalogName) { - xfer += oprot->writeFieldBegin("catalogName", ::apache::thrift::protocol::T_STRING, 8); + xfer += oprot->writeFieldBegin("catalogName", ::apache::thrift::protocol::T_STRING, 9); xfer += oprot->writeString(this->catalogName); xfer += oprot->writeFieldEnd(); } @@ -5337,6 +5377,7 @@ void swap(Database &a, Database &b) { swap(a.name, b.name); swap(a.description, b.description); swap(a.locationUri, b.locationUri); + swap(a.externalLocationUri, b.externalLocationUri); swap(a.parameters, b.parameters); swap(a.privileges, b.privileges); swap(a.ownerName, b.ownerName); @@ -5349,6 +5390,7 @@ Database::Database(const Database& other159) { name = other159.name; description = other159.description; locationUri = other159.locationUri; + externalLocationUri = other159.externalLocationUri; parameters = other159.parameters; privileges = other159.privileges; ownerName = other159.ownerName; @@ -5360,6 +5402,7 @@ Database& Database::operator=(const Database& other160) { name = other160.name; description = other160.description; locationUri = other160.locationUri; + externalLocationUri = other160.externalLocationUri; parameters = other160.parameters; privileges = other160.privileges; ownerName = other160.ownerName; @@ -5374,6 +5417,7 @@ void Database::printTo(std::ostream& out) const { out << "name=" << to_string(name); out << ", " << "description=" << to_string(description); out << ", " << "locationUri=" << to_string(locationUri); + out << ", " << "externalLocationUri="; (__isset.externalLocationUri ? (out << to_string(externalLocationUri)) : (out << "")); out << ", " << "parameters=" << to_string(parameters); out << ", " << "privileges="; (__isset.privileges ? (out << to_string(privileges)) : (out << "")); out << ", " << "ownerName="; (__isset.ownerName ? (out << to_string(ownerName)) : (out << "")); diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h index 5c6495e43fc2..91b8330306ed 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -2347,10 +2347,11 @@ inline std::ostream& operator<<(std::ostream& out, const GrantRevokeRoleResponse } typedef struct _Catalog__isset { - _Catalog__isset() : name(false), description(false), locationUri(false) {} + _Catalog__isset() : name(false), description(false), locationUri(false), externalLocationUri(false) {} bool name :1; bool description :1; bool locationUri :1; + bool externalLocationUri :1; } _Catalog__isset; class Catalog { @@ -2358,13 +2359,14 @@ class Catalog { Catalog(const Catalog&); Catalog& operator=(const Catalog&); - Catalog() : name(), description(), locationUri() { + Catalog() : name(), description(), locationUri(), externalLocationUri() { } virtual ~Catalog() throw(); std::string name; std::string description; std::string locationUri; + std::string externalLocationUri; _Catalog__isset __isset; @@ -2374,6 +2376,8 @@ class Catalog { void __set_locationUri(const std::string& val); + void __set_externalLocationUri(const std::string& val); + bool operator == (const Catalog & rhs) const { if (!(name == rhs.name)) @@ -2384,6 +2388,10 @@ class Catalog { return false; if (!(locationUri == rhs.locationUri)) return false; + if (__isset.externalLocationUri != rhs.__isset.externalLocationUri) + return false; + else if (__isset.externalLocationUri && !(externalLocationUri == rhs.externalLocationUri)) + return false; return true; } bool operator != (const Catalog &rhs) const { @@ -2689,10 +2697,11 @@ inline std::ostream& operator<<(std::ostream& out, const DropCatalogRequest& obj } typedef struct _Database__isset { - _Database__isset() : name(false), description(false), locationUri(false), parameters(false), privileges(false), ownerName(false), ownerType(false), catalogName(false) {} + _Database__isset() : name(false), description(false), locationUri(false), externalLocationUri(false), parameters(false), privileges(false), ownerName(false), ownerType(false), catalogName(false) {} bool name :1; bool description :1; bool locationUri :1; + bool externalLocationUri :1; bool parameters :1; bool privileges :1; bool ownerName :1; @@ -2705,13 +2714,14 @@ class Database { Database(const Database&); Database& operator=(const Database&); - Database() : name(), description(), locationUri(), ownerName(), ownerType((PrincipalType::type)0), catalogName() { + Database() : name(), description(), locationUri(), externalLocationUri(), ownerName(), ownerType((PrincipalType::type)0), catalogName() { } virtual ~Database() throw(); std::string name; std::string description; std::string locationUri; + std::string externalLocationUri; std::map parameters; PrincipalPrivilegeSet privileges; std::string ownerName; @@ -2726,6 +2736,8 @@ class Database { void __set_locationUri(const std::string& val); + void __set_externalLocationUri(const std::string& val); + void __set_parameters(const std::map & val); void __set_privileges(const PrincipalPrivilegeSet& val); @@ -2744,6 +2756,10 @@ class Database { return false; if (!(locationUri == rhs.locationUri)) return false; + if (__isset.externalLocationUri != rhs.__isset.externalLocationUri) + return false; + else if (__isset.externalLocationUri && !(externalLocationUri == rhs.externalLocationUri)) + return false; if (!(parameters == rhs.parameters)) return false; if (__isset.privileges != rhs.__isset.privileges) diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Catalog.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Catalog.java index 3eb4dbd51110..80249ead2578 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Catalog.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Catalog.java @@ -41,6 +41,7 @@ private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); private static final org.apache.thrift.protocol.TField DESCRIPTION_FIELD_DESC = new org.apache.thrift.protocol.TField("description", org.apache.thrift.protocol.TType.STRING, (short)2); private static final org.apache.thrift.protocol.TField LOCATION_URI_FIELD_DESC = new org.apache.thrift.protocol.TField("locationUri", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField EXTERNAL_LOCATION_URI_FIELD_DESC = new org.apache.thrift.protocol.TField("externalLocationUri", org.apache.thrift.protocol.TType.STRING, (short)4); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -51,12 +52,14 @@ private String name; // required private String description; // optional private String locationUri; // required + private String externalLocationUri; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { NAME((short)1, "name"), DESCRIPTION((short)2, "description"), - LOCATION_URI((short)3, "locationUri"); + LOCATION_URI((short)3, "locationUri"), + EXTERNAL_LOCATION_URI((short)4, "externalLocationUri"); private static final Map byName = new HashMap(); @@ -77,6 +80,8 @@ public static _Fields findByThriftId(int fieldId) { return DESCRIPTION; case 3: // LOCATION_URI return LOCATION_URI; + case 4: // EXTERNAL_LOCATION_URI + return EXTERNAL_LOCATION_URI; default: return null; } @@ -117,7 +122,7 @@ public String getFieldName() { } // isset id assignments - private static final _Fields optionals[] = {_Fields.DESCRIPTION}; + private static final _Fields optionals[] = {_Fields.DESCRIPTION,_Fields.EXTERNAL_LOCATION_URI}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -127,6 +132,8 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.LOCATION_URI, new org.apache.thrift.meta_data.FieldMetaData("locationUri", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.EXTERNAL_LOCATION_URI, new org.apache.thrift.meta_data.FieldMetaData("externalLocationUri", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Catalog.class, metaDataMap); } @@ -156,6 +163,9 @@ public Catalog(Catalog other) { if (other.isSetLocationUri()) { this.locationUri = other.locationUri; } + if (other.isSetExternalLocationUri()) { + this.externalLocationUri = other.externalLocationUri; + } } public Catalog deepCopy() { @@ -167,6 +177,7 @@ public void clear() { this.name = null; this.description = null; this.locationUri = null; + this.externalLocationUri = null; } public String getName() { @@ -238,6 +249,29 @@ public void setLocationUriIsSet(boolean value) { } } + public String getExternalLocationUri() { + return this.externalLocationUri; + } + + public void setExternalLocationUri(String externalLocationUri) { + this.externalLocationUri = externalLocationUri; + } + + public void unsetExternalLocationUri() { + this.externalLocationUri = null; + } + + /** Returns true if field externalLocationUri is set (has been assigned a value) and false otherwise */ + public boolean isSetExternalLocationUri() { + return this.externalLocationUri != null; + } + + public void setExternalLocationUriIsSet(boolean value) { + if (!value) { + this.externalLocationUri = null; + } + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case NAME: @@ -264,6 +298,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case EXTERNAL_LOCATION_URI: + if (value == null) { + unsetExternalLocationUri(); + } else { + setExternalLocationUri((String)value); + } + break; + } } @@ -278,6 +320,9 @@ public Object getFieldValue(_Fields field) { case LOCATION_URI: return getLocationUri(); + case EXTERNAL_LOCATION_URI: + return getExternalLocationUri(); + } throw new IllegalStateException(); } @@ -295,6 +340,8 @@ public boolean isSet(_Fields field) { return isSetDescription(); case LOCATION_URI: return isSetLocationUri(); + case EXTERNAL_LOCATION_URI: + return isSetExternalLocationUri(); } throw new IllegalStateException(); } @@ -339,6 +386,15 @@ public boolean equals(Catalog that) { return false; } + boolean this_present_externalLocationUri = true && this.isSetExternalLocationUri(); + boolean that_present_externalLocationUri = true && that.isSetExternalLocationUri(); + if (this_present_externalLocationUri || that_present_externalLocationUri) { + if (!(this_present_externalLocationUri && that_present_externalLocationUri)) + return false; + if (!this.externalLocationUri.equals(that.externalLocationUri)) + return false; + } + return true; } @@ -361,6 +417,11 @@ public int hashCode() { if (present_locationUri) list.add(locationUri); + boolean present_externalLocationUri = true && (isSetExternalLocationUri()); + list.add(present_externalLocationUri); + if (present_externalLocationUri) + list.add(externalLocationUri); + return list.hashCode(); } @@ -402,6 +463,16 @@ public int compareTo(Catalog other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetExternalLocationUri()).compareTo(other.isSetExternalLocationUri()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetExternalLocationUri()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.externalLocationUri, other.externalLocationUri); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -447,6 +518,16 @@ public String toString() { sb.append(this.locationUri); } first = false; + if (isSetExternalLocationUri()) { + if (!first) sb.append(", "); + sb.append("externalLocationUri:"); + if (this.externalLocationUri == null) { + sb.append("null"); + } else { + sb.append(this.externalLocationUri); + } + first = false; + } sb.append(")"); return sb.toString(); } @@ -514,6 +595,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Catalog struct) thr org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 4: // EXTERNAL_LOCATION_URI + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.externalLocationUri = iprot.readString(); + struct.setExternalLocationUriIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -544,6 +633,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, Catalog struct) th oprot.writeString(struct.locationUri); oprot.writeFieldEnd(); } + if (struct.externalLocationUri != null) { + if (struct.isSetExternalLocationUri()) { + oprot.writeFieldBegin(EXTERNAL_LOCATION_URI_FIELD_DESC); + oprot.writeString(struct.externalLocationUri); + oprot.writeFieldEnd(); + } + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -571,7 +667,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Catalog struct) thr if (struct.isSetLocationUri()) { optionals.set(2); } - oprot.writeBitSet(optionals, 3); + if (struct.isSetExternalLocationUri()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); if (struct.isSetName()) { oprot.writeString(struct.name); } @@ -581,12 +680,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Catalog struct) thr if (struct.isSetLocationUri()) { oprot.writeString(struct.locationUri); } + if (struct.isSetExternalLocationUri()) { + oprot.writeString(struct.externalLocationUri); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, Catalog struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(3); + BitSet incoming = iprot.readBitSet(4); if (incoming.get(0)) { struct.name = iprot.readString(); struct.setNameIsSet(true); @@ -599,6 +701,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Catalog struct) thro struct.locationUri = iprot.readString(); struct.setLocationUriIsSet(true); } + if (incoming.get(3)) { + struct.externalLocationUri = iprot.readString(); + struct.setExternalLocationUriIsSet(true); + } } } diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Database.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Database.java index 9cde9b869965..77b6ff3dc1ef 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Database.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Database.java @@ -41,11 +41,12 @@ private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); private static final org.apache.thrift.protocol.TField DESCRIPTION_FIELD_DESC = new org.apache.thrift.protocol.TField("description", org.apache.thrift.protocol.TType.STRING, (short)2); private static final org.apache.thrift.protocol.TField LOCATION_URI_FIELD_DESC = new org.apache.thrift.protocol.TField("locationUri", org.apache.thrift.protocol.TType.STRING, (short)3); - private static final org.apache.thrift.protocol.TField PARAMETERS_FIELD_DESC = new org.apache.thrift.protocol.TField("parameters", org.apache.thrift.protocol.TType.MAP, (short)4); - private static final org.apache.thrift.protocol.TField PRIVILEGES_FIELD_DESC = new org.apache.thrift.protocol.TField("privileges", org.apache.thrift.protocol.TType.STRUCT, (short)5); - private static final org.apache.thrift.protocol.TField OWNER_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("ownerName", org.apache.thrift.protocol.TType.STRING, (short)6); - private static final org.apache.thrift.protocol.TField OWNER_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("ownerType", org.apache.thrift.protocol.TType.I32, (short)7); - private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)8); + private static final org.apache.thrift.protocol.TField EXTERNAL_LOCATION_URI_FIELD_DESC = new org.apache.thrift.protocol.TField("externalLocationUri", org.apache.thrift.protocol.TType.STRING, (short)4); + private static final org.apache.thrift.protocol.TField PARAMETERS_FIELD_DESC = new org.apache.thrift.protocol.TField("parameters", org.apache.thrift.protocol.TType.MAP, (short)5); + private static final org.apache.thrift.protocol.TField PRIVILEGES_FIELD_DESC = new org.apache.thrift.protocol.TField("privileges", org.apache.thrift.protocol.TType.STRUCT, (short)6); + private static final org.apache.thrift.protocol.TField OWNER_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("ownerName", org.apache.thrift.protocol.TType.STRING, (short)7); + private static final org.apache.thrift.protocol.TField OWNER_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("ownerType", org.apache.thrift.protocol.TType.I32, (short)8); + private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)9); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -56,6 +57,7 @@ private String name; // required private String description; // required private String locationUri; // required + private String externalLocationUri; // optional private Map parameters; // required private PrincipalPrivilegeSet privileges; // optional private String ownerName; // optional @@ -67,15 +69,16 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { NAME((short)1, "name"), DESCRIPTION((short)2, "description"), LOCATION_URI((short)3, "locationUri"), - PARAMETERS((short)4, "parameters"), - PRIVILEGES((short)5, "privileges"), - OWNER_NAME((short)6, "ownerName"), + EXTERNAL_LOCATION_URI((short)4, "externalLocationUri"), + PARAMETERS((short)5, "parameters"), + PRIVILEGES((short)6, "privileges"), + OWNER_NAME((short)7, "ownerName"), /** * * @see PrincipalType */ - OWNER_TYPE((short)7, "ownerType"), - CATALOG_NAME((short)8, "catalogName"); + OWNER_TYPE((short)8, "ownerType"), + CATALOG_NAME((short)9, "catalogName"); private static final Map byName = new HashMap(); @@ -96,15 +99,17 @@ public static _Fields findByThriftId(int fieldId) { return DESCRIPTION; case 3: // LOCATION_URI return LOCATION_URI; - case 4: // PARAMETERS + case 4: // EXTERNAL_LOCATION_URI + return EXTERNAL_LOCATION_URI; + case 5: // PARAMETERS return PARAMETERS; - case 5: // PRIVILEGES + case 6: // PRIVILEGES return PRIVILEGES; - case 6: // OWNER_NAME + case 7: // OWNER_NAME return OWNER_NAME; - case 7: // OWNER_TYPE + case 8: // OWNER_TYPE return OWNER_TYPE; - case 8: // CATALOG_NAME + case 9: // CATALOG_NAME return CATALOG_NAME; default: return null; @@ -146,7 +151,7 @@ public String getFieldName() { } // isset id assignments - private static final _Fields optionals[] = {_Fields.PRIVILEGES,_Fields.OWNER_NAME,_Fields.OWNER_TYPE,_Fields.CATALOG_NAME}; + private static final _Fields optionals[] = {_Fields.EXTERNAL_LOCATION_URI,_Fields.PRIVILEGES,_Fields.OWNER_NAME,_Fields.OWNER_TYPE,_Fields.CATALOG_NAME}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -156,6 +161,8 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.LOCATION_URI, new org.apache.thrift.meta_data.FieldMetaData("locationUri", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.EXTERNAL_LOCATION_URI, new org.apache.thrift.meta_data.FieldMetaData("externalLocationUri", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.PARAMETERS, new org.apache.thrift.meta_data.FieldMetaData("parameters", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), @@ -201,6 +208,9 @@ public Database(Database other) { if (other.isSetLocationUri()) { this.locationUri = other.locationUri; } + if (other.isSetExternalLocationUri()) { + this.externalLocationUri = other.externalLocationUri; + } if (other.isSetParameters()) { Map __this__parameters = new HashMap(other.parameters); this.parameters = __this__parameters; @@ -228,6 +238,7 @@ public void clear() { this.name = null; this.description = null; this.locationUri = null; + this.externalLocationUri = null; this.parameters = null; this.privileges = null; this.ownerName = null; @@ -304,6 +315,29 @@ public void setLocationUriIsSet(boolean value) { } } + public String getExternalLocationUri() { + return this.externalLocationUri; + } + + public void setExternalLocationUri(String externalLocationUri) { + this.externalLocationUri = externalLocationUri; + } + + public void unsetExternalLocationUri() { + this.externalLocationUri = null; + } + + /** Returns true if field externalLocationUri is set (has been assigned a value) and false otherwise */ + public boolean isSetExternalLocationUri() { + return this.externalLocationUri != null; + } + + public void setExternalLocationUriIsSet(boolean value) { + if (!value) { + this.externalLocationUri = null; + } + } + public int getParametersSize() { return (this.parameters == null) ? 0 : this.parameters.size(); } @@ -464,6 +498,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case EXTERNAL_LOCATION_URI: + if (value == null) { + unsetExternalLocationUri(); + } else { + setExternalLocationUri((String)value); + } + break; + case PARAMETERS: if (value == null) { unsetParameters(); @@ -518,6 +560,9 @@ public Object getFieldValue(_Fields field) { case LOCATION_URI: return getLocationUri(); + case EXTERNAL_LOCATION_URI: + return getExternalLocationUri(); + case PARAMETERS: return getParameters(); @@ -550,6 +595,8 @@ public boolean isSet(_Fields field) { return isSetDescription(); case LOCATION_URI: return isSetLocationUri(); + case EXTERNAL_LOCATION_URI: + return isSetExternalLocationUri(); case PARAMETERS: return isSetParameters(); case PRIVILEGES: @@ -604,6 +651,15 @@ public boolean equals(Database that) { return false; } + boolean this_present_externalLocationUri = true && this.isSetExternalLocationUri(); + boolean that_present_externalLocationUri = true && that.isSetExternalLocationUri(); + if (this_present_externalLocationUri || that_present_externalLocationUri) { + if (!(this_present_externalLocationUri && that_present_externalLocationUri)) + return false; + if (!this.externalLocationUri.equals(that.externalLocationUri)) + return false; + } + boolean this_present_parameters = true && this.isSetParameters(); boolean that_present_parameters = true && that.isSetParameters(); if (this_present_parameters || that_present_parameters) { @@ -671,6 +727,11 @@ public int hashCode() { if (present_locationUri) list.add(locationUri); + boolean present_externalLocationUri = true && (isSetExternalLocationUri()); + list.add(present_externalLocationUri); + if (present_externalLocationUri) + list.add(externalLocationUri); + boolean present_parameters = true && (isSetParameters()); list.add(present_parameters); if (present_parameters) @@ -737,6 +798,16 @@ public int compareTo(Database other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetExternalLocationUri()).compareTo(other.isSetExternalLocationUri()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetExternalLocationUri()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.externalLocationUri, other.externalLocationUri); + if (lastComparison != 0) { + return lastComparison; + } + } lastComparison = Boolean.valueOf(isSetParameters()).compareTo(other.isSetParameters()); if (lastComparison != 0) { return lastComparison; @@ -830,6 +901,16 @@ public String toString() { sb.append(this.locationUri); } first = false; + if (isSetExternalLocationUri()) { + if (!first) sb.append(", "); + sb.append("externalLocationUri:"); + if (this.externalLocationUri == null) { + sb.append("null"); + } else { + sb.append(this.externalLocationUri); + } + first = false; + } if (!first) sb.append(", "); sb.append("parameters:"); if (this.parameters == null) { @@ -948,7 +1029,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Database struct) th org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 4: // PARAMETERS + case 4: // EXTERNAL_LOCATION_URI + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.externalLocationUri = iprot.readString(); + struct.setExternalLocationUriIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // PARAMETERS if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { { org.apache.thrift.protocol.TMap _map102 = iprot.readMapBegin(); @@ -968,7 +1057,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Database struct) th org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 5: // PRIVILEGES + case 6: // PRIVILEGES if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { struct.privileges = new PrincipalPrivilegeSet(); struct.privileges.read(iprot); @@ -977,7 +1066,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Database struct) th org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 6: // OWNER_NAME + case 7: // OWNER_NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { struct.ownerName = iprot.readString(); struct.setOwnerNameIsSet(true); @@ -985,7 +1074,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Database struct) th org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 7: // OWNER_TYPE + case 8: // OWNER_TYPE if (schemeField.type == org.apache.thrift.protocol.TType.I32) { struct.ownerType = org.apache.hadoop.hive.metastore.api.PrincipalType.findByValue(iprot.readI32()); struct.setOwnerTypeIsSet(true); @@ -993,7 +1082,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Database struct) th org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 8: // CATALOG_NAME + case 9: // CATALOG_NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { struct.catalogName = iprot.readString(); struct.setCatalogNameIsSet(true); @@ -1029,6 +1118,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, Database struct) t oprot.writeString(struct.locationUri); oprot.writeFieldEnd(); } + if (struct.externalLocationUri != null) { + if (struct.isSetExternalLocationUri()) { + oprot.writeFieldBegin(EXTERNAL_LOCATION_URI_FIELD_DESC); + oprot.writeString(struct.externalLocationUri); + oprot.writeFieldEnd(); + } + } if (struct.parameters != null) { oprot.writeFieldBegin(PARAMETERS_FIELD_DESC); { @@ -1097,22 +1193,25 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Database struct) th if (struct.isSetLocationUri()) { optionals.set(2); } - if (struct.isSetParameters()) { + if (struct.isSetExternalLocationUri()) { optionals.set(3); } - if (struct.isSetPrivileges()) { + if (struct.isSetParameters()) { optionals.set(4); } - if (struct.isSetOwnerName()) { + if (struct.isSetPrivileges()) { optionals.set(5); } - if (struct.isSetOwnerType()) { + if (struct.isSetOwnerName()) { optionals.set(6); } - if (struct.isSetCatalogName()) { + if (struct.isSetOwnerType()) { optionals.set(7); } - oprot.writeBitSet(optionals, 8); + if (struct.isSetCatalogName()) { + optionals.set(8); + } + oprot.writeBitSet(optionals, 9); if (struct.isSetName()) { oprot.writeString(struct.name); } @@ -1122,6 +1221,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Database struct) th if (struct.isSetLocationUri()) { oprot.writeString(struct.locationUri); } + if (struct.isSetExternalLocationUri()) { + oprot.writeString(struct.externalLocationUri); + } if (struct.isSetParameters()) { { oprot.writeI32(struct.parameters.size()); @@ -1149,7 +1251,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Database struct) th @Override public void read(org.apache.thrift.protocol.TProtocol prot, Database struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(8); + BitSet incoming = iprot.readBitSet(9); if (incoming.get(0)) { struct.name = iprot.readString(); struct.setNameIsSet(true); @@ -1163,6 +1265,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Database struct) thr struct.setLocationUriIsSet(true); } if (incoming.get(3)) { + struct.externalLocationUri = iprot.readString(); + struct.setExternalLocationUriIsSet(true); + } + if (incoming.get(4)) { { org.apache.thrift.protocol.TMap _map108 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); struct.parameters = new HashMap(2*_map108.size); @@ -1177,20 +1283,20 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Database struct) thr } struct.setParametersIsSet(true); } - if (incoming.get(4)) { + if (incoming.get(5)) { struct.privileges = new PrincipalPrivilegeSet(); struct.privileges.read(iprot); struct.setPrivilegesIsSet(true); } - if (incoming.get(5)) { + if (incoming.get(6)) { struct.ownerName = iprot.readString(); struct.setOwnerNameIsSet(true); } - if (incoming.get(6)) { + if (incoming.get(7)) { struct.ownerType = org.apache.hadoop.hive.metastore.api.PrincipalType.findByValue(iprot.readI32()); struct.setOwnerTypeIsSet(true); } - if (incoming.get(7)) { + if (incoming.get(8)) { struct.catalogName = iprot.readString(); struct.setCatalogNameIsSet(true); } diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php index a29ebb7f5985..c8db47e7076d 100644 --- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -4495,6 +4495,10 @@ class Catalog { * @var string */ public $locationUri = null; + /** + * @var string + */ + public $externalLocationUri = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -4511,6 +4515,10 @@ public function __construct($vals=null) { 'var' => 'locationUri', 'type' => TType::STRING, ), + 4 => array( + 'var' => 'externalLocationUri', + 'type' => TType::STRING, + ), ); } if (is_array($vals)) { @@ -4523,6 +4531,9 @@ public function __construct($vals=null) { if (isset($vals['locationUri'])) { $this->locationUri = $vals['locationUri']; } + if (isset($vals['externalLocationUri'])) { + $this->externalLocationUri = $vals['externalLocationUri']; + } } } @@ -4566,6 +4577,13 @@ public function read($input) $xfer += $input->skip($ftype); } break; + case 4: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->externalLocationUri); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -4594,6 +4612,11 @@ public function write($output) { $xfer += $output->writeString($this->locationUri); $xfer += $output->writeFieldEnd(); } + if ($this->externalLocationUri !== null) { + $xfer += $output->writeFieldBegin('externalLocationUri', TType::STRING, 4); + $xfer += $output->writeString($this->externalLocationUri); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; @@ -5130,6 +5153,10 @@ class Database { * @var string */ public $locationUri = null; + /** + * @var string + */ + public $externalLocationUri = null; /** * @var array */ @@ -5167,6 +5194,10 @@ public function __construct($vals=null) { 'type' => TType::STRING, ), 4 => array( + 'var' => 'externalLocationUri', + 'type' => TType::STRING, + ), + 5 => array( 'var' => 'parameters', 'type' => TType::MAP, 'ktype' => TType::STRING, @@ -5178,20 +5209,20 @@ public function __construct($vals=null) { 'type' => TType::STRING, ), ), - 5 => array( + 6 => array( 'var' => 'privileges', 'type' => TType::STRUCT, 'class' => '\metastore\PrincipalPrivilegeSet', ), - 6 => array( + 7 => array( 'var' => 'ownerName', 'type' => TType::STRING, ), - 7 => array( + 8 => array( 'var' => 'ownerType', 'type' => TType::I32, ), - 8 => array( + 9 => array( 'var' => 'catalogName', 'type' => TType::STRING, ), @@ -5207,6 +5238,9 @@ public function __construct($vals=null) { if (isset($vals['locationUri'])) { $this->locationUri = $vals['locationUri']; } + if (isset($vals['externalLocationUri'])) { + $this->externalLocationUri = $vals['externalLocationUri']; + } if (isset($vals['parameters'])) { $this->parameters = $vals['parameters']; } @@ -5266,6 +5300,13 @@ public function read($input) } break; case 4: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->externalLocationUri); + } else { + $xfer += $input->skip($ftype); + } + break; + case 5: if ($ftype == TType::MAP) { $this->parameters = array(); $_size90 = 0; @@ -5285,7 +5326,7 @@ public function read($input) $xfer += $input->skip($ftype); } break; - case 5: + case 6: if ($ftype == TType::STRUCT) { $this->privileges = new \metastore\PrincipalPrivilegeSet(); $xfer += $this->privileges->read($input); @@ -5293,21 +5334,21 @@ public function read($input) $xfer += $input->skip($ftype); } break; - case 6: + case 7: if ($ftype == TType::STRING) { $xfer += $input->readString($this->ownerName); } else { $xfer += $input->skip($ftype); } break; - case 7: + case 8: if ($ftype == TType::I32) { $xfer += $input->readI32($this->ownerType); } else { $xfer += $input->skip($ftype); } break; - case 8: + case 9: if ($ftype == TType::STRING) { $xfer += $input->readString($this->catalogName); } else { @@ -5342,11 +5383,16 @@ public function write($output) { $xfer += $output->writeString($this->locationUri); $xfer += $output->writeFieldEnd(); } + if ($this->externalLocationUri !== null) { + $xfer += $output->writeFieldBegin('externalLocationUri', TType::STRING, 4); + $xfer += $output->writeString($this->externalLocationUri); + $xfer += $output->writeFieldEnd(); + } if ($this->parameters !== null) { if (!is_array($this->parameters)) { throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA); } - $xfer += $output->writeFieldBegin('parameters', TType::MAP, 4); + $xfer += $output->writeFieldBegin('parameters', TType::MAP, 5); { $output->writeMapBegin(TType::STRING, TType::STRING, count($this->parameters)); { @@ -5364,22 +5410,22 @@ public function write($output) { if (!is_object($this->privileges)) { throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA); } - $xfer += $output->writeFieldBegin('privileges', TType::STRUCT, 5); + $xfer += $output->writeFieldBegin('privileges', TType::STRUCT, 6); $xfer += $this->privileges->write($output); $xfer += $output->writeFieldEnd(); } if ($this->ownerName !== null) { - $xfer += $output->writeFieldBegin('ownerName', TType::STRING, 6); + $xfer += $output->writeFieldBegin('ownerName', TType::STRING, 7); $xfer += $output->writeString($this->ownerName); $xfer += $output->writeFieldEnd(); } if ($this->ownerType !== null) { - $xfer += $output->writeFieldBegin('ownerType', TType::I32, 7); + $xfer += $output->writeFieldBegin('ownerType', TType::I32, 8); $xfer += $output->writeI32($this->ownerType); $xfer += $output->writeFieldEnd(); } if ($this->catalogName !== null) { - $xfer += $output->writeFieldBegin('catalogName', TType::STRING, 8); + $xfer += $output->writeFieldBegin('catalogName', TType::STRING, 9); $xfer += $output->writeString($this->catalogName); $xfer += $output->writeFieldEnd(); } diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 11affe375b07..4d0290da1ac2 100644 --- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -3253,6 +3253,7 @@ class Catalog: - name - description - locationUri + - externalLocationUri """ thrift_spec = ( @@ -3260,12 +3261,14 @@ class Catalog: (1, TType.STRING, 'name', None, None, ), # 1 (2, TType.STRING, 'description', None, None, ), # 2 (3, TType.STRING, 'locationUri', None, None, ), # 3 + (4, TType.STRING, 'externalLocationUri', None, None, ), # 4 ) - def __init__(self, name=None, description=None, locationUri=None,): + def __init__(self, name=None, description=None, locationUri=None, externalLocationUri=None,): self.name = name self.description = description self.locationUri = locationUri + self.externalLocationUri = externalLocationUri def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -3291,6 +3294,11 @@ def read(self, iprot): self.locationUri = iprot.readString() else: iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.externalLocationUri = iprot.readString() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -3313,6 +3321,10 @@ def write(self, oprot): oprot.writeFieldBegin('locationUri', TType.STRING, 3) oprot.writeString(self.locationUri) oprot.writeFieldEnd() + if self.externalLocationUri is not None: + oprot.writeFieldBegin('externalLocationUri', TType.STRING, 4) + oprot.writeString(self.externalLocationUri) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -3325,6 +3337,7 @@ def __hash__(self): value = (value * 31) ^ hash(self.name) value = (value * 31) ^ hash(self.description) value = (value * 31) ^ hash(self.locationUri) + value = (value * 31) ^ hash(self.externalLocationUri) return value def __repr__(self): @@ -3758,6 +3771,7 @@ class Database: - name - description - locationUri + - externalLocationUri - parameters - privileges - ownerName @@ -3770,17 +3784,19 @@ class Database: (1, TType.STRING, 'name', None, None, ), # 1 (2, TType.STRING, 'description', None, None, ), # 2 (3, TType.STRING, 'locationUri', None, None, ), # 3 - (4, TType.MAP, 'parameters', (TType.STRING,None,TType.STRING,None), None, ), # 4 - (5, TType.STRUCT, 'privileges', (PrincipalPrivilegeSet, PrincipalPrivilegeSet.thrift_spec), None, ), # 5 - (6, TType.STRING, 'ownerName', None, None, ), # 6 - (7, TType.I32, 'ownerType', None, None, ), # 7 - (8, TType.STRING, 'catalogName', None, None, ), # 8 + (4, TType.STRING, 'externalLocationUri', None, None, ), # 4 + (5, TType.MAP, 'parameters', (TType.STRING,None,TType.STRING,None), None, ), # 5 + (6, TType.STRUCT, 'privileges', (PrincipalPrivilegeSet, PrincipalPrivilegeSet.thrift_spec), None, ), # 6 + (7, TType.STRING, 'ownerName', None, None, ), # 7 + (8, TType.I32, 'ownerType', None, None, ), # 8 + (9, TType.STRING, 'catalogName', None, None, ), # 9 ) - def __init__(self, name=None, description=None, locationUri=None, parameters=None, privileges=None, ownerName=None, ownerType=None, catalogName=None,): + def __init__(self, name=None, description=None, locationUri=None, externalLocationUri=None, parameters=None, privileges=None, ownerName=None, ownerType=None, catalogName=None,): self.name = name self.description = description self.locationUri = locationUri + self.externalLocationUri = externalLocationUri self.parameters = parameters self.privileges = privileges self.ownerName = ownerName @@ -3812,6 +3828,11 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 4: + if ftype == TType.STRING: + self.externalLocationUri = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 5: if ftype == TType.MAP: self.parameters = {} (_ktype91, _vtype92, _size90 ) = iprot.readMapBegin() @@ -3822,23 +3843,23 @@ def read(self, iprot): iprot.readMapEnd() else: iprot.skip(ftype) - elif fid == 5: + elif fid == 6: if ftype == TType.STRUCT: self.privileges = PrincipalPrivilegeSet() self.privileges.read(iprot) else: iprot.skip(ftype) - elif fid == 6: + elif fid == 7: if ftype == TType.STRING: self.ownerName = iprot.readString() else: iprot.skip(ftype) - elif fid == 7: + elif fid == 8: if ftype == TType.I32: self.ownerType = iprot.readI32() else: iprot.skip(ftype) - elif fid == 8: + elif fid == 9: if ftype == TType.STRING: self.catalogName = iprot.readString() else: @@ -3865,8 +3886,12 @@ def write(self, oprot): oprot.writeFieldBegin('locationUri', TType.STRING, 3) oprot.writeString(self.locationUri) oprot.writeFieldEnd() + if self.externalLocationUri is not None: + oprot.writeFieldBegin('externalLocationUri', TType.STRING, 4) + oprot.writeString(self.externalLocationUri) + oprot.writeFieldEnd() if self.parameters is not None: - oprot.writeFieldBegin('parameters', TType.MAP, 4) + oprot.writeFieldBegin('parameters', TType.MAP, 5) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) for kiter97,viter98 in self.parameters.items(): oprot.writeString(kiter97) @@ -3874,19 +3899,19 @@ def write(self, oprot): oprot.writeMapEnd() oprot.writeFieldEnd() if self.privileges is not None: - oprot.writeFieldBegin('privileges', TType.STRUCT, 5) + oprot.writeFieldBegin('privileges', TType.STRUCT, 6) self.privileges.write(oprot) oprot.writeFieldEnd() if self.ownerName is not None: - oprot.writeFieldBegin('ownerName', TType.STRING, 6) + oprot.writeFieldBegin('ownerName', TType.STRING, 7) oprot.writeString(self.ownerName) oprot.writeFieldEnd() if self.ownerType is not None: - oprot.writeFieldBegin('ownerType', TType.I32, 7) + oprot.writeFieldBegin('ownerType', TType.I32, 8) oprot.writeI32(self.ownerType) oprot.writeFieldEnd() if self.catalogName is not None: - oprot.writeFieldBegin('catalogName', TType.STRING, 8) + oprot.writeFieldBegin('catalogName', TType.STRING, 9) oprot.writeString(self.catalogName) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -3901,6 +3926,7 @@ def __hash__(self): value = (value * 31) ^ hash(self.name) value = (value * 31) ^ hash(self.description) value = (value * 31) ^ hash(self.locationUri) + value = (value * 31) ^ hash(self.externalLocationUri) value = (value * 31) ^ hash(self.parameters) value = (value * 31) ^ hash(self.privileges) value = (value * 31) ^ hash(self.ownerName) diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb index fc640d07c119..4401d6ea947c 100644 --- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -789,11 +789,13 @@ class Catalog NAME = 1 DESCRIPTION = 2 LOCATIONURI = 3 + EXTERNALLOCATIONURI = 4 FIELDS = { NAME => {:type => ::Thrift::Types::STRING, :name => 'name'}, DESCRIPTION => {:type => ::Thrift::Types::STRING, :name => 'description', :optional => true}, - LOCATIONURI => {:type => ::Thrift::Types::STRING, :name => 'locationUri'} + LOCATIONURI => {:type => ::Thrift::Types::STRING, :name => 'locationUri'}, + EXTERNALLOCATIONURI => {:type => ::Thrift::Types::STRING, :name => 'externalLocationUri', :optional => true} } def struct_fields; FIELDS; end @@ -907,16 +909,18 @@ class Database NAME = 1 DESCRIPTION = 2 LOCATIONURI = 3 - PARAMETERS = 4 - PRIVILEGES = 5 - OWNERNAME = 6 - OWNERTYPE = 7 - CATALOGNAME = 8 + EXTERNALLOCATIONURI = 4 + PARAMETERS = 5 + PRIVILEGES = 6 + OWNERNAME = 7 + OWNERTYPE = 8 + CATALOGNAME = 9 FIELDS = { NAME => {:type => ::Thrift::Types::STRING, :name => 'name'}, DESCRIPTION => {:type => ::Thrift::Types::STRING, :name => 'description'}, LOCATIONURI => {:type => ::Thrift::Types::STRING, :name => 'locationUri'}, + EXTERNALLOCATIONURI => {:type => ::Thrift::Types::STRING, :name => 'externalLocationUri', :optional => true}, PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}, PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true}, OWNERNAME => {:type => ::Thrift::Types::STRING, :name => 'ownerName', :optional => true}, diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 1327fa267f6e..3f6ee10beb15 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -31,6 +31,7 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependNotNullCatToDbName; import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; @@ -77,6 +78,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.*; @@ -198,7 +200,8 @@ public class HiveMetaStore extends ThriftHiveMetastore { public static final Logger LOG = LoggerFactory.getLogger(HiveMetaStore.class); public static final String PARTITION_NUMBER_EXCEED_LIMIT_MSG = "Number of partitions scanned (=%d) on table '%s' exceeds limit (=%d). This is controlled on the metastore server by %s."; - + private static final FsPermission EXTERNAL_DB_DIRECTORIES_PERMISSIONS + = FsPermission.createImmutable((short)0700); // boolean that tells if the HiveMetaStore (remote) server is being used. // Can be used to determine if the calls to metastore api (HMSHandler) are being made with // embedded metastore or a remote one @@ -733,6 +736,7 @@ public static void createDefaultCatalog(RawStore ms, Warehouse wh) throws MetaEx // script does not know the location of the warehouse. So we need to update it. LOG.info("Setting location of default catalog, as it hasn't been done after upgrade"); defaultCat.setLocationUri(wh.getWhRoot().toString()); + defaultCat.setExternalLocationUri(wh.getWhRootExternal().toString()); ms.alterCatalog(defaultCat.getName(), defaultCat); } @@ -747,8 +751,11 @@ private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObje try { ms.getDatabase(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME); } catch (NoSuchObjectException e) { + wh.maybeCreateExternalWarehouseDir(); + Database db = new Database(DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT, wh.getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null); + db.setExternalLocationUri(wh.getDefaultExternalDatabasePath(DEFAULT_DATABASE_NAME).toString()); db.setOwnerName(PUBLIC); db.setOwnerType(PrincipalType.ROLE); db.setCatalogName(DEFAULT_CATALOG_NAME); @@ -1034,6 +1041,7 @@ public void create_catalog(CreateCatalogRequest rqst) // Create a default database inside the catalog Database db = new Database(DEFAULT_DATABASE_NAME, "Default database for catalog " + catalog.getName(), catalog.getLocationUri(), Collections.emptyMap()); + db.setExternalLocationUri(catalog.getExternalLocationUri()); db.setCatalogName(catalog.getName()); create_database_core(ms, db); @@ -1255,20 +1263,70 @@ private void create_database_core(RawStore ms, final Database db) throw new InvalidObjectException("No such catalog " + db.getCatalogName()); } Path dbPath = wh.determineDatabasePath(cat, db); + Path dbExternalPath = wh.determineExternalDatabasePath(cat, db); db.setLocationUri(dbPath.toString()); + db.setExternalLocationUri(dbExternalPath.toString()); boolean success = false; - boolean madeDir = false; + boolean madeManagedDir = false; + boolean madeExternalDir = false; Map transactionalListenersResponses = Collections.emptyMap(); try { firePreEvent(new PreCreateDatabaseEvent(db, this)); - if (!wh.isDir(dbPath)) { - LOG.debug("Creating database path " + dbPath); - if (!wh.mkdirs(dbPath)) { - throw new MetaException("Unable to create database path " + dbPath + - ", failed to create database " + db.getName()); + + + try { + // Since this may be done as random user (if doAs=true) he may not have access + // to the managed directory. We run this as an admin user + madeManagedDir = UserGroupInformation.getLoginUser().doAs( + new PrivilegedExceptionAction() { + @Override + public Boolean run() throws Exception { + if (!wh.isDir(dbPath)) { + LOG.info("Creating database path in managed directory " + dbPath); + if (!wh.mkdirs(dbPath)) { + throw new MetaException("Unable to create database managed path " + dbPath + + ", failed to create database " + db.getName()); + } + return true; + } + return false; + } + }); + if (madeManagedDir) { + LOG.info("Created database path in managed directory " + dbExternalPath); } - madeDir = true; + } catch (IOException | InterruptedException e) { + throw new MetaException("Unable to create database external path " + dbExternalPath + + ", failed to create database " + db.getName()); + } + + try { + madeExternalDir = UserGroupInformation.getCurrentUser().doAs( + new PrivilegedExceptionAction() { + @Override + public Boolean run() { + try { + if (!wh.isDir(dbExternalPath)) { + LOG.info("Creating database path in external directory " + dbExternalPath); + return wh.mkdirs(dbExternalPath, EXTERNAL_DB_DIRECTORIES_PERMISSIONS); + } + } catch (MetaException e) { + LOG.warn("Failed to create external path " + dbExternalPath + " for database " + db.getName() + + ". Access won't be allowed if the StorageBasedAuthorizationProvider is enabled ", e); + } + return false; + } + }); + if (madeExternalDir) { + LOG.info("Created external path " + dbExternalPath + " for database " + db.getName()); + } else { + LOG.warn("Failed to create external path " + dbExternalPath + " for database " + db.getName() + + ". Access won't be allowed if the StorageBasedAuthorizationProvider is enabled "); + } + } catch (IOException | InterruptedException | UndeclaredThrowableException e) { + LOG.warn("Failed to create external path " + dbExternalPath + " for database " + db.getName() + + ". Access won't be allowed if the StorageBasedAuthorizationProvider is enabled ", e); } ms.openTransaction(); @@ -1285,8 +1343,37 @@ private void create_database_core(RawStore ms, final Database db) } finally { if (!success) { ms.rollbackTransaction(); - if (madeDir) { - wh.deleteDir(dbPath, true, db); + + if (madeManagedDir) { + try { + UserGroupInformation.getLoginUser().doAs( + new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + wh.deleteDir(dbPath, true, db); + return null; + } + }); + } catch (IOException | InterruptedException e) { + LOG.error("Couldn't delete managed directory " + dbPath + " after " + + "it was created for database " + db.getName()); + } + } + + if (madeExternalDir) { + try { + UserGroupInformation.getCurrentUser().doAs( + new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + wh.deleteDir(dbExternalPath, true, db); + return null; + } + }); + } catch (IOException | InterruptedException e) { + LOG.warn("Couldn't delete external directory " + dbExternalPath + " after " + + "it was created for database " + db.getName()); + } } } @@ -1388,6 +1475,11 @@ public void alter_database(final String dbName, final Database newDB) throws TEx newDB.setLocationUri(wh.getDnsPath(new Path(newDB.getLocationUri())).toString()); } + if (newDB.isSetExternalLocationUri()) { + newDB.setExternalLocationUri(wh.getDnsPath(new Path(newDB.getExternalLocationUri())) + .toString()); + } + String[] parsedDbName = parseDbName(dbName, conf); try { @@ -1588,12 +1680,37 @@ private void drop_database_core(RawStore ms, String catName, } // Delete the data in the database try { - wh.deleteDir(new Path(db.getLocationUri()), true, db); - } catch (Exception e) { - LOG.error("Failed to delete database directory: " + db.getLocationUri() + - " " + e.getMessage()); + final Database dbFinal = db; + Boolean deleted = UserGroupInformation.getLoginUser().doAs( + new PrivilegedExceptionAction() { + @Override + public Boolean run() throws Exception { + return wh.deleteDir(new Path(dbFinal.getLocationUri()), true, dbFinal); + } + }); + if (!deleted) { + LOG.warn("Failed to delete database folder " + db.getLocationUri()); + } + } catch (IOException | InterruptedException | UndeclaredThrowableException e) { + LOG.warn("Might have failed to delete database folder " + db.getLocationUri()); } // it is not a terrible thing even if the data is not deleted + + try { + if (db.isSetExternalLocationUri()) { + final Path externalPath = new Path(db.getExternalLocationUri()); + Boolean deleted = UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction() { + @Override public Boolean run() throws Exception { + return wh.deleteDirIfEmpty(externalPath); + } + }); + if (!deleted) { + LOG.warn("Failed to delete database external folder " + db.getExternalLocationUri()); + } + } + } catch (IOException | InterruptedException e) { + LOG.warn("Might have failed to delete database external folder " + db.getExternalLocationUri()); + } } if (!listeners.isEmpty()) { diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index f45b71fb1fb8..db1c383066f0 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -327,8 +327,8 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ catName = catName.toLowerCase(); String queryTextDbSelector= "select " - + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", " - + "\"OWNER_NAME\", \"OWNER_TYPE\", \"CTLG_NAME\" " + + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DB_EXTERNAL_LOCATION_URI\", " + + " \"DESC\", \"OWNER_NAME\", \"OWNER_TYPE\", \"CTLG_NAME\" " + "FROM "+ DBS + " where \"NAME\" = ? and \"CTLG_NAME\" = ? "; Object[] params = new Object[] { dbName, catName }; @@ -375,17 +375,19 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ Database db = new Database(); db.setName(extractSqlString(dbline[1])); db.setLocationUri(extractSqlString(dbline[2])); - db.setDescription(extractSqlString(dbline[3])); - db.setOwnerName(extractSqlString(dbline[4])); - String type = extractSqlString(dbline[5]); + db.setExternalLocationUri(extractSqlString(dbline[3])); + db.setDescription(extractSqlString(dbline[4])); + db.setOwnerName(extractSqlString(dbline[5])); + String type = extractSqlString(dbline[6]); db.setOwnerType( (null == type || type.trim().isEmpty()) ? null : PrincipalType.valueOf(type)); - db.setCatalogName(extractSqlString(dbline[6])); + db.setCatalogName(extractSqlString(dbline[7])); db.setParameters(MetaStoreUtils.trimMapNulls(dbParams,convertMapNullsToEmptyStrings)); if (LOG.isDebugEnabled()){ LOG.debug("getDatabase: directsql returning db " + db.getName() - + " locn["+db.getLocationUri() +"] desc [" +db.getDescription() - + "] owner [" + db.getOwnerName() + "] ownertype ["+ db.getOwnerType() +"]"); + + " locn["+db.getLocationUri() +"] "+ " extLocn["+db.getExternalLocationUri() +"] " + + " desc [" +db.getDescription() + "] owner [" + db.getOwnerName() + "]" + + " ownertype ["+ db.getOwnerType() +"]"); } return db; } finally { diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 0d2da7a200eb..63400bd5fd40 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -838,6 +838,9 @@ public void alterCatalog(String catName, Catalog cat) if (org.apache.commons.lang.StringUtils.isNotBlank(cat.getLocationUri())) { mCat.setLocationUri(cat.getLocationUri()); } + if (org.apache.commons.lang.StringUtils.isNotBlank(cat.getExternalLocationUri())) { + mCat.setLocationUri(cat.getExternalLocationUri()); + } if (org.apache.commons.lang.StringUtils.isNotBlank(cat.getDescription())) { mCat.setDescription(cat.getDescription()); } @@ -928,11 +931,13 @@ private MCatalog catToMCat(Catalog cat) { mCat.setDescription(cat.getDescription()); } mCat.setLocationUri(cat.getLocationUri()); + mCat.setExternalLocationUri(cat.getExternalLocationUri()); return mCat; } private Catalog mCatToCat(MCatalog mCat) { Catalog cat = new Catalog(mCat.getName(), mCat.getLocationUri()); + cat.setExternalLocationUri(mCat.getExternalLocationUri()); if (mCat.getDescription() != null) { cat.setDescription(mCat.getDescription()); } @@ -948,6 +953,7 @@ public void createDatabase(Database db) throws InvalidObjectException, MetaExcep assert mdb.getCatalogName() != null; mdb.setName(db.getName().toLowerCase()); mdb.setLocationUri(db.getLocationUri()); + mdb.setExternalLocationUri(db.getExternalLocationUri()); mdb.setDescription(db.getDescription()); mdb.setParameters(db.getParameters()); mdb.setOwnerName(db.getOwnerName()); @@ -1040,6 +1046,7 @@ public Database getJDODatabase(String catName, String name) throws NoSuchObjectE db.setName(mdb.getName()); db.setDescription(mdb.getDescription()); db.setLocationUri(mdb.getLocationUri()); + db.setExternalLocationUri(mdb.getExternalLocationUri()); db.setParameters(convertMap(mdb.getParameters())); db.setOwnerName(mdb.getOwnerName()); String type = org.apache.commons.lang.StringUtils.defaultIfBlank(mdb.getOwnerType(), null); @@ -1076,6 +1083,9 @@ public boolean alterDatabase(String catName, String dbName, Database db) if (org.apache.commons.lang.StringUtils.isNotBlank(db.getLocationUri())) { mdb.setLocationUri(db.getLocationUri()); } + if (org.apache.commons.lang.StringUtils.isNotBlank(db.getExternalLocationUri())) { + mdb.setExternalLocationUri(db.getExternalLocationUri()); + } openTransaction(); pm.makePersistent(mdb); committed = commitTransaction(); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java index da5a71cc64dd..97c6adbc80b3 100755 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -26,11 +26,13 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -60,6 +62,9 @@ * This class represents a warehouse where data of Hive tables is stored */ public class Warehouse { + private static final String DEFAULT_EXTERNAL_DIRECTORY = ".external"; + private static final FsPermission DEFAULT_EXTERNAL_DIRECTORY_PERMISSIONS + = FsPermission.createImmutable((short)0777); public static final String DEFAULT_CATALOG_NAME = "hive"; public static final String DEFAULT_CATALOG_COMMENT = "Default catalog, for Hive"; public static final String DEFAULT_DATABASE_NAME = "default"; @@ -72,7 +77,7 @@ public class Warehouse { private Path whRootExternal; private final Configuration conf; private final String whRootString; - private final String whRootExternalString; + private String whRootExternalString; public static final Logger LOG = LoggerFactory.getLogger("hive.metastore.warehouse"); @@ -148,6 +153,11 @@ public Path getDnsPath(Path path) throws MetaException { return getDnsPath(path, conf); } + public Path getHomeDir() throws IOException { + FileSystem fs =FileSystem.get(conf); + return fs.getHomeDirectory(); + } + /** * Resolve the configured warehouse root dir with respect to the configuration * This involves opening the FileSystem corresponding to the warehouse root @@ -174,6 +184,22 @@ public Path getWhRootExternal() throws MetaException { return whRootExternal; } + void maybeCreateExternalWarehouseDir() throws MetaException{ + if (!hasExternalWarehouseRoot()) { + Path externalDir = null; + try { + externalDir = new Path(getHomeDir(), DEFAULT_EXTERNAL_DIRECTORY); + } catch (IOException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + whRootExternal = externalDir; + whRootExternalString = externalDir.toString(); + if (!(isDir(externalDir))) { + mkdirs(externalDir, DEFAULT_EXTERNAL_DIRECTORY_PERMISSIONS); + } + } + } + /** * Build the database path based on catalog name and database name. This should only be used * when a database is being created or altered. If you just want to find out the path a @@ -201,6 +227,38 @@ public Path determineDatabasePath(Catalog cat, Database db) throws MetaException } } + /** + * Build the database external path based on catalog name and database name. This should only be used + * when a database is being created or altered. If you just want to find out the path a + * database is already using call {@link #getDatabaseExternalPath(Database)}. If the passed in + * database already has a path set that will be used. If not the location will be built using + * catalog's path and the database name. + * @param cat catalog the database is in + * @param db database object + * @return Path representing the directory for the database + * @throws MetaException when the file path cannot be properly determined from the configured + * file system. + */ + public Path determineExternalDatabasePath(Catalog cat, Database db) throws MetaException { + if (db.isSetExternalLocationUri()) { + return getDnsPath(new Path(db.getExternalLocationUri())); + } + if (cat == null || cat.getName().equalsIgnoreCase(DEFAULT_CATALOG_NAME)) { + if (db.getName().equalsIgnoreCase(DEFAULT_DATABASE_NAME)) { + return getWhRootExternal(); + } else { + return new Path(getWhRootExternal(), dbDirFromDbName(db)); + } + } else { + if (cat.isSetExternalLocationUri()) { + return new Path(getDnsPath(new Path(cat.getExternalLocationUri())), dbDirFromDbName(db)); + } + LOG.warn("Default external directory should be set in catalog or in the database, setting: " + + getWhRootExternal() + " for database " + db.getName()); + return new Path(getWhRootExternal(), dbDirFromDbName(db)); + } + } + private String dbDirFromDbName(Database db) throws MetaException { return db.getName().toLowerCase() + DATABASE_WAREHOUSE_SUFFIX; } @@ -221,6 +279,25 @@ public Path getDatabasePath(Database db) throws MetaException { return new Path(db.getLocationUri()); } + /** + * Get the external path specified by the database. In the case of the default database the root of the + * external warehouse is returned. + * @param db database to get the external path of + * @return external path to the database directory + * @throws MetaException when the file path cannot be properly determined from the configured + * file system. + */ + public Optional getDatabaseExternalPath(Database db) throws MetaException { + if (db.getCatalogName().equalsIgnoreCase(DEFAULT_CATALOG_NAME) && + db.getName().equalsIgnoreCase(DEFAULT_DATABASE_NAME)) { + return Optional.of(getWhRootExternal()); + } + if (db.isSetExternalLocationUri()) { + return Optional.of(new Path(db.getExternalLocationUri())); + } + return Optional.empty(); + } + public Path getDefaultDatabasePath(String dbName) throws MetaException { // TODO CAT - I am fairly certain that most calls to this are in error. This should only be // used when the database location is unset, which should never happen except when a @@ -318,6 +395,17 @@ public boolean mkdirs(Path f) throws MetaException { return false; } + public boolean mkdirs(Path f, FsPermission permission) throws MetaException { + FileSystem fs; + try { + fs = getFs(f); + return FileUtils.mkdir(fs, f, permission); + } catch (IOException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + return false; + } + public boolean renameDir(Path sourcePath, Path destPath, boolean needCmRecycle) throws MetaException { try { if (needCmRecycle) { @@ -342,6 +430,16 @@ void addToChangeManagement(Path file) throws MetaException { } } + public boolean deleteDirIfEmpty(Path f) throws MetaException, IOException { + FileSystem fs = getFs(f); + if (FileUtils.isDirEmpty(fs, f)) { + return deleteDir(f, false, false, false); + } else { + LOG.info("Will not delete external directory " + f + " since it's not empty"); + } + return true; + } + public boolean deleteDir(Path f, boolean recursive, Database db) throws MetaException { return deleteDir(f, recursive, false, db); } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/CatalogBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/CatalogBuilder.java index be76d937b7fc..a23da7ee0d6b 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/CatalogBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/CatalogBuilder.java @@ -23,7 +23,7 @@ import org.apache.thrift.TException; public class CatalogBuilder { - private String name, description, location; + private String name, description, location, externalLocation; public CatalogBuilder setName(String name) { this.name = name; @@ -40,10 +40,16 @@ public CatalogBuilder setLocation(String location) { return this; } + public CatalogBuilder setExternalLocation(String externalLocation) { + this.externalLocation = externalLocation; + return this; + } + public Catalog build() throws MetaException { if (name == null) throw new MetaException("You must name the catalog"); if (location == null) throw new MetaException("You must give the catalog a location"); Catalog catalog = new Catalog(name, location); + catalog.setExternalLocationUri(externalLocation); if (description != null) catalog.setDescription(description); return catalog; } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java index f3d2182a04ab..035f4d1a9c9a 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java @@ -37,7 +37,7 @@ * selects reasonable defaults. */ public class DatabaseBuilder { - private String name, description, location, catalogName; + private String name, description, location, externalLocation, catalogName; private Map params = new HashMap<>(); private String ownerName; private PrincipalType ownerType; @@ -70,6 +70,11 @@ public DatabaseBuilder setLocation(String location) { return this; } + public DatabaseBuilder setExternalLocation(String externalLocation) { + this.externalLocation = externalLocation; + return this; + } + public DatabaseBuilder setParams(Map params) { this.params = params; return this; @@ -100,6 +105,7 @@ public Database build(Configuration conf) throws MetaException { db.setOwnerName(ownerName); if (ownerType == null) ownerType = PrincipalType.USER; db.setOwnerType(ownerType); + db.setExternalLocationUri(externalLocation); return db; } catch (IOException e) { throw MetaStoreUtils.newMetaException(e); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MCatalog.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MCatalog.java index e82cb4322f6e..7ebd6759281c 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MCatalog.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MCatalog.java @@ -21,6 +21,7 @@ public class MCatalog { private String name; private String description; private String locationUri; + private String externalLocationUri; public MCatalog() { @@ -55,4 +56,12 @@ public String getLocationUri() { public void setLocationUri(String locationUri) { this.locationUri = locationUri; } + + public String getExternalLocationUri() { + return externalLocationUri; + } + + public void setExternalLocationUri(String externalLocationUri) { + this.externalLocationUri = externalLocationUri; + } } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MDatabase.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MDatabase.java index 815b39c483b2..55ca463ab894 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MDatabase.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MDatabase.java @@ -35,6 +35,7 @@ public class MDatabase { private String name; private String locationUri; + private String externalLocationUri; private String description; private Map parameters; private String ownerName; @@ -154,4 +155,12 @@ public String getCatalogName() { public void setCatalogName(String catalogName) { this.catalogName = catalogName; } + + public String getExternalLocationUri() { + return externalLocationUri; + } + + public void setExternalLocationUri(String externalLocationUri) { + this.externalLocationUri = externalLocationUri; + } } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java index 963e12f9d858..7fe383d86d5a 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java @@ -26,14 +26,18 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; import java.util.ArrayList; import java.util.BitSet; import java.util.Collections; @@ -168,6 +172,25 @@ public static boolean mkdir(FileSystem fs, Path f) throws IOException { return fs.mkdirs(f); } + /** + * Creates the directory and all necessary parent directories. + * @param fs FileSystem to use + * @param f path to create. + * @param fsPermission permissions to use. + * @return true if directory created successfully. False otherwise, including if it exists. + * @throws IOException exception in creating the directory + */ + public static boolean mkdir(FileSystem fs, Path f, FsPermission fsPermission) throws IOException { + LOG.info("Creating directory if it doesn't exist: " + f + + " , with permissions: " + fsPermission); + // We need to create it in two steps, otherwise we may get different permissions + if(fs.mkdirs(f)) { + fs.setPermission(f, fsPermission); + return true; + } + return false; + } + /** * Rename a file. Unlike {@link FileSystem#rename(Path, Path)}, if the destPath already exists * and is a directory, this will NOT move the sourcePath into it. It will throw an IOException diff --git a/standalone-metastore/src/main/resources/package.jdo b/standalone-metastore/src/main/resources/package.jdo index 1be3e986a505..39921e57c06f 100644 --- a/standalone-metastore/src/main/resources/package.jdo +++ b/standalone-metastore/src/main/resources/package.jdo @@ -47,6 +47,9 @@ + + + diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql index 352b43e9cd71..9dc43b4d2df1 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql @@ -19,6 +19,7 @@ CREATE TABLE "APP"."DBS" ( "DB_ID" BIGINT NOT NULL, "DESC" VARCHAR(4000), "DB_LOCATION_URI" VARCHAR(4000) NOT NULL, + "DB_EXTERNAL_LOCATION_URI" VARCHAR(4000), "NAME" VARCHAR(128), "OWNER_NAME" VARCHAR(128), "OWNER_TYPE" VARCHAR(10), diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql index bb691053a58c..bfbb9e0e7c28 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -19,6 +19,7 @@ CREATE TABLE "APP"."DBS" ( "DB_ID" BIGINT NOT NULL, "DESC" VARCHAR(4000), "DB_LOCATION_URI" VARCHAR(4000) NOT NULL, + "DB_EXTERNAL_LOCATION_URI" VARCHAR(4000), "NAME" VARCHAR(128), "OWNER_NAME" VARCHAR(128), "OWNER_TYPE" VARCHAR(10), diff --git a/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql index 6621ef7ce510..13a53eb74000 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql @@ -29,5 +29,7 @@ ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer; CREATE INDEX "APP"."TAB_COL_STATS_IDX" ON "APP"."TAB_COL_STATS" ("CAT_NAME", "DB_NAME", "TABLE_NAME", "COLUMN_NAME"); +ALTER TABLE "APP"."DBS" ADD COLUMN "DB_EXTERNAL_LOCATION_URI" VARCHAR(4000); + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql index bc11b4092adf..f99b811dfd92 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql @@ -282,6 +282,7 @@ CREATE TABLE DBS DB_ID bigint NOT NULL, "DESC" nvarchar(4000) NULL, DB_LOCATION_URI nvarchar(4000) NOT NULL, + DB_EXTERNAL_LOCATION_URI nvarchar(4000) NULL, "NAME" nvarchar(128) NULL, OWNER_NAME nvarchar(128) NULL, OWNER_TYPE nvarchar(10) NULL, diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 922e8fef3891..bedf6e3c8244 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -283,6 +283,7 @@ CREATE TABLE DBS DB_ID bigint NOT NULL, "DESC" nvarchar(4000) NULL, DB_LOCATION_URI nvarchar(4000) NOT NULL, + DB_EXTERNAL_LOCATION_URI nvarchar(4000) NULL, "NAME" nvarchar(128) NULL, OWNER_NAME nvarchar(128) NULL, OWNER_TYPE nvarchar(10) NULL, diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql index abb80d6c54ce..3a5a7cce489c 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql @@ -30,6 +30,8 @@ ALTER TABLE TXNS ADD TXN_TYPE int NULL; CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, COLUMN_NAME); +ALTER TABLE PART_COL_STATS ADD DB_EXTERNAL_LOCATION_URI nvarchar(4000) NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS MESSAGE; diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql index af955dcf7f04..35a0836b8188 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql @@ -96,6 +96,7 @@ CREATE TABLE IF NOT EXISTS `DBS` ( `DB_ID` bigint(20) NOT NULL, `DESC` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `DB_LOCATION_URI` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `DB_EXTERNAL_LOCATION_URI` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `OWNER_TYPE` varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 6c40e6e4f666..869843051a6a 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -96,6 +96,7 @@ CREATE TABLE IF NOT EXISTS `DBS` ( `DB_ID` bigint(20) NOT NULL, `DESC` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `DB_LOCATION_URI` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `DB_EXTERNAL_LOCATION_URI` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `OWNER_TYPE` varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql index 305fa1d67d45..aed74dccb863 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql @@ -30,6 +30,8 @@ ALTER TABLE TXNS ADD COLUMN TXN_TYPE int DEFAULT NULL; CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, COLUMN_NAME) USING BTREE; +ALTER TABLE `DBS` ADD COLUMN `DB_EXTERNAL_LOCATION_URI` VARCHAR(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS ' '; diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql index bc137039c6c5..8ff7c1c51688 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql @@ -89,6 +89,7 @@ CREATE TABLE DBS DB_ID NUMBER NOT NULL, "DESC" VARCHAR2(4000) NULL, DB_LOCATION_URI VARCHAR2(4000) NOT NULL, + DB_EXTERNAL_LOCATION_URI VARCHAR2(4000) NULL, "NAME" VARCHAR2(128) NULL, OWNER_NAME VARCHAR2(128) NULL, OWNER_TYPE VARCHAR2(10) NULL, diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index e12150a43816..e8a7f56ad66e 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -89,6 +89,7 @@ CREATE TABLE DBS DB_ID NUMBER NOT NULL, "DESC" VARCHAR2(4000) NULL, DB_LOCATION_URI VARCHAR2(4000) NOT NULL, + DB_EXTERNAL_LOCATION_URI VARCHAR2(4000) NULL, "NAME" VARCHAR2(128) NULL, OWNER_NAME VARCHAR2(128) NULL, OWNER_TYPE VARCHAR2(10) NULL, diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql index ccdea54ab352..f4a5595f2839 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql @@ -30,6 +30,8 @@ ALTER TABLE TXNS ADD TXN_TYPE number(10) NULL; CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, COLUMN_NAME); +ALTER TABLE DBS ADD DB_EXTERNAL_LOCATION_URI VARCHAR2(4000) NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS Status from dual; diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql index 36bab70eda5c..400b1dc04947 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql @@ -74,6 +74,7 @@ CREATE TABLE "DBS" ( "DB_ID" bigint NOT NULL, "DESC" character varying(4000) DEFAULT NULL::character varying, "DB_LOCATION_URI" character varying(4000) NOT NULL, + "DB_EXTERNAL_LOCATION_URI" character varying(4000) DEFAULT NULL::character varying, "NAME" character varying(128) DEFAULT NULL::character varying, "OWNER_NAME" character varying(128) DEFAULT NULL::character varying, "OWNER_TYPE" character varying(10) DEFAULT NULL::character varying, diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index b73e1d19f6f1..8add94ee39c6 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -74,6 +74,7 @@ CREATE TABLE "DBS" ( "DB_ID" bigint NOT NULL, "DESC" character varying(4000) DEFAULT NULL::character varying, "DB_LOCATION_URI" character varying(4000) NOT NULL, + "DB_EXTERNAL_LOCATION_URI" character varying(4000) DEFAULT NULL::character varying, "NAME" character varying(128) DEFAULT NULL::character varying, "OWNER_NAME" character varying(128) DEFAULT NULL::character varying, "OWNER_TYPE" character varying(10) DEFAULT NULL::character varying, diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql index 2c0eb31aadb5..01202f352a00 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql @@ -32,6 +32,8 @@ ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer DEFAULT NULL; CREATE INDEX "TAB_COL_STATS_IDX" ON "TAB_COL_STATS" USING btree ("CAT_NAME", "DB_NAME","TABLE_NAME","COLUMN_NAME"); +ALTER TABLE "DBS" ADD COLUMN "DB_EXTERNAL_LOCATION_URI" character varying(4000) DEFAULT NULL::character varying; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.1.0', "VERSION_COMMENT"='Hive release version 3.1.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0'; diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index 6e503eb9082d..a19a829c3a6f 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -147,7 +147,7 @@ enum PartitionEventType { LOAD_DONE = 1, } -// Enums for transaction and lock management +// Enums for transaction and lock management enum TxnState { COMMITTED = 1, ABORTED = 2, @@ -327,11 +327,14 @@ struct GrantRevokeRoleResponse { } struct Catalog { - 1: string name, // Name of the catalog - 2: optional string description, // description of the catalog - 3: string locationUri // default storage location. When databases are created in - // this catalog, if they do not specify a location, they will - // be placed in this location. + 1: string name, // Name of the catalog + 2: optional string description, // description of the catalog + 3: string locationUri, // default storage location. When databases are created in + // this catalog, if they do not specify a location, they will + // be placed in this location. + 4: optional string externalLocationUri // same as locationUri but for external tables. If it's not + // specified the external default warehouse directory will be + // used. } struct CreateCatalogRequest { @@ -364,11 +367,12 @@ struct Database { 1: string name, 2: string description, 3: string locationUri, - 4: map parameters, // properties associated with the database - 5: optional PrincipalPrivilegeSet privileges, - 6: optional string ownerName, - 7: optional PrincipalType ownerType, - 8: optional string catalogName + 4: optional string externalLocationUri, // optional since it can't be specified when creating the database + 5: map parameters, // properties associated with the database + 6: optional PrincipalPrivilegeSet privileges, + 7: optional string ownerName, + 8: optional PrincipalType ownerType, + 9: optional string catalogName } // This object holds the information needed by SerDes @@ -656,7 +660,7 @@ struct CheckConstraintsResponse { struct DropConstraintRequest { - 1: required string dbname, + 1: required string dbname, 2: required string tablename, 3: required string constraintname, 4: optional string catName @@ -1122,7 +1126,7 @@ struct FireEventRequest { struct FireEventResponse { // NOP for now, this is just a place holder for future responses } - + struct MetadataPpdResult { 1: optional binary metadata, 2: optional binary includeBitset @@ -1669,7 +1673,7 @@ service ThriftHiveMetastore extends fb303.FacebookService void add_primary_key(1:AddPrimaryKeyRequest req) throws(1:NoSuchObjectException o1, 2:MetaException o2) void add_foreign_key(1:AddForeignKeyRequest req) - throws(1:NoSuchObjectException o1, 2:MetaException o2) + throws(1:NoSuchObjectException o1, 2:MetaException o2) void add_unique_constraint(1:AddUniqueConstraintRequest req) throws(1:NoSuchObjectException o1, 2:MetaException o2) void add_not_null_constraint(1:AddNotNullConstraintRequest req) @@ -2093,13 +2097,13 @@ service ThriftHiveMetastore extends fb303.FacebookService ShowLocksResponse show_locks(1:ShowLocksRequest rqst) void heartbeat(1:HeartbeatRequest ids) throws (1:NoSuchLockException o1, 2:NoSuchTxnException o2, 3:TxnAbortedException o3) HeartbeatTxnRangeResponse heartbeat_txn_range(1:HeartbeatTxnRangeRequest txns) - void compact(1:CompactionRequest rqst) - CompactionResponse compact2(1:CompactionRequest rqst) + void compact(1:CompactionRequest rqst) + CompactionResponse compact2(1:CompactionRequest rqst) ShowCompactResponse show_compact(1:ShowCompactRequest rqst) void add_dynamic_partitions(1:AddDynamicPartitions rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2) // Notification logging calls - NotificationEventResponse get_next_notification(1:NotificationEventRequest rqst) + NotificationEventResponse get_next_notification(1:NotificationEventRequest rqst) CurrentNotificationEventId get_current_notificationEventId() NotificationEventsCountResponse get_notification_events_count(1:NotificationEventsCountRequest rqst) FireEventResponse fire_listener_event(1:FireEventRequest rqst) @@ -2202,7 +2206,7 @@ service ThriftHiveMetastore extends fb303.FacebookService LockResponse get_lock_materialization_rebuild(1: string dbName, 2: string tableName, 3: i64 txnId) bool heartbeat_lock_materialization_rebuild(1: string dbName, 2: string tableName, 3: i64 txnId) - + void add_runtime_stats(1: RuntimeStat stat) throws(1:MetaException o1) list get_runtime_stats(1: GetRuntimeStatsRequest rqst) throws(1:MetaException o1) } diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index cb32236d548e..aba03310b53a 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; +import java.io.FileNotFoundException; import java.lang.reflect.Field; import java.io.IOException; import java.sql.Connection; @@ -40,6 +41,7 @@ import java.util.concurrent.TimeUnit; import com.google.common.collect.Sets; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; @@ -47,6 +49,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.datanucleus.api.jdo.JDOPersistenceManager; import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; import org.junit.Assert; @@ -1069,6 +1072,50 @@ public void testDatabaseLocationWithPermissionProblems() throws Exception { assertTrue("Database creation succeeded even with permission problem", createFailed); } + @Test + public void testExternalDirectory() throws Exception{ + String externalDirString = MetastoreConf.getVar(conf, ConfVars.WAREHOUSE_EXTERNAL); + Path externalDir = new Path(externalDirString); + silentDropDatabase(TEST_DB1_NAME); + + String dbLocation = + MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/test/_testDB_create_"; + + String dbExternalLocation = externalDirString + "/test/_testDB_create_"; + + + FileSystem fs = FileSystem.get(new Path(dbLocation).toUri(), conf); + fs.mkdirs( + new Path(MetastoreConf.getVar(conf, ConfVars.WAREHOUSE)), + new FsPermission((short) 700)); + + fs.mkdirs(externalDir, new FsPermission((short) 0777)); + + Database db = new DatabaseBuilder() + .setName(TEST_DB1_NAME) + .setLocation(dbLocation) + .setExternalLocation(dbExternalLocation) + .build(conf); + client.createDatabase(db); + FileStatus fileStatus = fs.getFileStatus(new Path(dbExternalLocation)); + + assertTrue("External folder should have been created", fileStatus.isDirectory()); + assertEquals("External folder should have the right permissions", new FsPermission((short) 0700), + fileStatus.getPermission()); + assertEquals("External folder should be owned by the right username", + UserGroupInformation.getCurrentUser().getShortUserName(), fileStatus.getOwner()); + client.dropDatabase(db.getName()); + + try { + fs.getFileStatus(new Path(dbExternalLocation)); + fail("External directory should have been deleted"); + } catch (FileNotFoundException e ) {} + finally { + fs.delete(new Path(MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/test"), true); + fs.delete(new Path(MetastoreConf.getVar(conf, ConfVars.WAREHOUSE_EXTERNAL) + "/test"), true); + } + } + @Test public void testDatabaseLocation() throws Throwable { try { diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index 5d5bc76e37e8..51bc15afb0d9 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -218,6 +218,7 @@ public void testTableOps() throws MetaException, InvalidObjectException, NoSuchO .setName(DB1) .setDescription("description") .setLocation("locationurl") + .setExternalLocation("externalurl") .build(conf); objectStore.createDatabase(db1); StorageDescriptor sd1 = @@ -316,6 +317,7 @@ public void testPartitionOps() throws MetaException, InvalidObjectException, .setName(DB1) .setDescription("description") .setLocation("locationurl") + .setExternalLocation("externalurl") .build(conf); objectStore.createDatabase(db1); StorageDescriptor sd = createFakeSd("location"); @@ -485,6 +487,7 @@ private void createPartitionedTable(boolean withPrivileges, boolean withStatisti .setName(DB1) .setDescription("description") .setLocation("locationurl") + .setExternalLocation("externalurl") .build(conf); objectStore.createDatabase(db1); Table tbl1 = diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java index a15f5ea0453c..54302caa6ff6 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java @@ -523,7 +523,7 @@ public void testAddPartitionForExternalTableNullLocation() throws Exception { client.getPartition(DB_NAME, tableName, Lists.newArrayList(DEFAULT_YEAR_VALUE)); Assert.assertNotNull(resultPart); Assert.assertNotNull(resultPart.getSd()); - String defaultTableLocation = metaStore.getWarehouseRoot() + "/" + DB_NAME + ".db/" + tableName; + String defaultTableLocation = metaStore.getExternalWarehouseRoot() + "/" + DB_NAME + ".db/" + tableName; String defaulPartitionLocation = defaultTableLocation + "/year=2017"; Assert.assertEquals(defaulPartitionLocation, resultPart.getSd().getLocation()); } @@ -1197,7 +1197,7 @@ public void testAddPartitionsForExternalTableNullLocation() throws Exception { Lists.newArrayList("year=2017", "year=2018")); Assert.assertNotNull(resultParts); Assert.assertEquals(2, resultParts.size()); - String defaultTableLocation = metaStore.getWarehouseRoot() + "/" + DB_NAME + ".db/" + tableName; + String defaultTableLocation = metaStore.getExternalWarehouseRoot() + "/" + DB_NAME + ".db/" + tableName; String defaultPartLocation1 = defaultTableLocation + "/year=2017"; String defaultPartLocation2 = defaultTableLocation + "/year=2018"; if (resultParts.get(0).getValues().get(0).equals("2017")) { diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java index d323ac6c90ed..d4db952ce262 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java @@ -430,6 +430,7 @@ public void testAlterDatabase() throws Exception { .setOwnerType(PrincipalType.GROUP) .setOwnerName("owner2") .setLocation(metaStore.getWarehouseRoot() + "/database_location_2") + .setExternalLocation(metaStore.getExternalWarehouseRoot() + "/database_location_2") .setDescription("dummy description 2") .addParam("param_key_1", "param_value_1_2") .addParam("param_key_2_3", "param_value_2_3") @@ -626,6 +627,7 @@ private Database getDatabaseWithAllParametersSet() throws Exception { .setOwnerType(PrincipalType.ROLE) .setOwnerName("owner") .setLocation(metaStore.getWarehouseRoot() + "/database_location") + .setExternalLocation(metaStore.getExternalWarehouseRoot() + "/database_location") .setDescription("dummy description") .addParam("param_key_1", "param_value_1") .addParam("param_key_2", "param_value_2") diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java index 709085d71ff8..ec0162f4f107 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java @@ -113,6 +113,16 @@ public Path getWarehouseRoot() throws MetaException { return warehouse.getWhRoot(); } + /** + * Returns the MetaStore Warehouse root directory name. + * + * @return The warehouse root directory + * @throws MetaException IO failure + */ + public Path getExternalWarehouseRoot() throws MetaException { + return warehouse.getWhRootExternal(); + } + /** * Check if a path exists. * diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java index 9e425cff0612..e0475a076663 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java @@ -100,7 +100,7 @@ public void testValidateSequences() throws Exception { String[] scripts = new String[] { "insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100);", "insert into CTLGS values(37, 'mycat', 'my description', 'hdfs://tmp');", - "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'mycat');" + "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'hdfs:///tmp2', 'db1', 'test', 'test', 'mycat');" }; File scriptFile = generateTestScript(scripts); schemaTool.runSqlLine(scriptFile.getPath()); @@ -112,7 +112,7 @@ public void testValidateSequences() throws Exception { "delete from SEQUENCE_TABLE;", "delete from DBS;", "insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100);", - "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'mycat');" + "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'hdfs:///tmp2', 'db1', 'test', 'test', 'mycat');" }; scriptFile = generateTestScript(scripts); schemaTool.runSqlLine(scriptFile.getPath()); @@ -339,8 +339,8 @@ public void testValidateLocations() throws Exception { // Test valid case String[] scripts = new String[] { "insert into CTLGS values (1, 'mycat', 'mydescription', 'hdfs://myhost.com:8020/user/hive/warehouse');", - "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');", - "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'mycat');", + "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'hdfs://myhost.com:8020/user/hive/external/mydb', 'mydb', 'public', 'role', 'mycat');", + "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'hdfs://myhost.com:8020/external', 'haDB', 'public', 'role', 'mycat');", "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", @@ -367,10 +367,10 @@ public void testValidateLocations() throws Exception { "delete from TBLS;", "delete from SDS;", "delete from DBS;", - "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');", - "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'mycat');", - "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'mycat');", - "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'mycat');", + "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', '/user/hive/external/mydb', 'mydb', 'public', 'role', 'mycat');", + "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', 'hdfs://myhost.com:8020/external', '', 'public', 'role', 'mycat');", + "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'hdfs://myhost.com:8020/external', 'zDB', 'public', 'role', 'mycat');", + "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'hdfs://mynameservice.com/external', 'haDB', 'public', 'role', 'mycat');", "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", @@ -468,7 +468,7 @@ private String writeDummyPreUpgradeScript(int index, String upgradeScriptName, private void createTestHiveTableSchemas() throws IOException { String[] scripts = new String[] { "insert into CTLGS values (1, 'mycat', 'my description', 'hdfs://myhost.com:8020/user/hive/warehouse');", - "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');", + "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'hdfs://myhost.com:8020/user/hive/external/mydb', 'mydb', 'public', 'role', 'mycat');", "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');",