Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

adapter: Change id column in mz_databases and mz_schemas #18719

Merged
merged 5 commits into from
Apr 17, 2023
Merged

adapter: Change id column in mz_databases and mz_schemas #18719

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1d81b99
start
ParkMyCar Apr 12, 2023
6f81b5b
Fix upgrades
ParkMyCar Apr 13, 2023
f6c33ae
review feedback, rename methods, remove footguns, add comments
ParkMyCar Apr 14, 2023
9b89be9
get rid of the KeyJson types
ParkMyCar Apr 14, 2023
efdcdd7
Update src/stash/src/lib.rs
do-not-use-parker-timmerman Apr 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
39 changes: 33 additions & 6 deletions src/adapter/src/catalog.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1353,7 +1353,14 @@ impl CatalogState {
/// that the serialized state for two identical catalogs will compare
/// identically.
pub fn dump(&self) -> String {
serde_json::to_string(&self.database_by_id).expect("serialization cannot fail")
// Note: database_by_id is a Map whose keys are not Strings, but serializing
// a Map to JSON requires the keys be strings, hence the mapping here.
let database_by_str: BTreeMap<String, _> = self
.database_by_id
.iter()
.map(|(key, value)| (key.to_string(), value.debug_json()))
.collect();
serde_json::to_string(&database_by_str).expect("serialization cannot fail")
}

pub fn availability_zones(&self) -> &[String] {
Expand Down Expand Up @@ -1517,6 +1524,26 @@ pub struct Database {
pub owner_id: RoleId,
}

impl Database {
/// Returns a `Database` formatted as a `serde_json::Value` that is suitable for debugging. For
/// example `CatalogState::dump`.
fn debug_json(&self) -> serde_json::Value {
let schemas_by_str: BTreeMap<String, _> = self
.schemas_by_id
.iter()
.map(|(key, value)| (key.to_string(), value))
.collect();

serde_json::json!({
"name": self.name,
"id": self.id,
"schemas_by_id": schemas_by_str,
"schemas_by_name": self.schemas_by_name,
"owner_id": self.owner_id,
})
}
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Schema {
pub name: QualifiedSchemaName,
Expand Down Expand Up @@ -4774,8 +4801,8 @@ impl Catalog {
public_schema_oid,
owner_id,
} => {
let database_id = tx.insert_database(&name, owner_id)?;
let schema_id = tx.insert_schema(database_id, DEFAULT_SCHEMA, owner_id)?;
let database_id = tx.insert_user_database(&name, owner_id)?;
let schema_id = tx.insert_user_schema(database_id, DEFAULT_SCHEMA, owner_id)?;
state.add_to_audit_log(
oracle_write_ts,
session,
Expand Down Expand Up @@ -4850,7 +4877,7 @@ impl Catalog {
)));
}
};
let schema_id = tx.insert_schema(database_id, &schema_name, owner_id)?;
let schema_id = tx.insert_user_schema(database_id, &schema_name, owner_id)?;
state.add_to_audit_log(
oracle_write_ts,
session,
Expand Down Expand Up @@ -8030,8 +8057,8 @@ mod tests {
item,
name: QualifiedItemName {
qualifiers: ItemQualifiers {
database_spec: ResolvedDatabaseSpecifier::Id(DatabaseId::new(1)),
schema_spec: SchemaSpecifier::Id(SchemaId::new(3)),
database_spec: ResolvedDatabaseSpecifier::Id(DatabaseId::User(1)),
schema_spec: SchemaSpecifier::Id(SchemaId::User(3)),
},
item: "v".to_string(),
},
Expand Down
24 changes: 12 additions & 12 deletions src/adapter/src/catalog/builtin.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1383,7 +1383,7 @@ pub static MZ_DATABASES: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
name: "mz_databases",
schema: MZ_CATALOG_SCHEMA,
desc: RelationDesc::empty()
.with_column("id", ScalarType::UInt64.nullable(false))
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column("owner_id", ScalarType::String.nullable(false)),
Expand All @@ -1393,9 +1393,9 @@ pub static MZ_SCHEMAS: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
name: "mz_schemas",
schema: MZ_CATALOG_SCHEMA,
desc: RelationDesc::empty()
.with_column("id", ScalarType::UInt64.nullable(false))
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("database_id", ScalarType::UInt64.nullable(true))
.with_column("database_id", ScalarType::String.nullable(true))
.with_column("name", ScalarType::String.nullable(false))
.with_column("owner_id", ScalarType::String.nullable(false)),
is_retained_metrics_object: false,
Expand Down Expand Up @@ -1442,7 +1442,7 @@ pub static MZ_TABLES: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
desc: RelationDesc::empty()
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("schema_id", ScalarType::UInt64.nullable(false))
.with_column("schema_id", ScalarType::String.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column("owner_id", ScalarType::String.nullable(false)),
is_retained_metrics_object: false,
Expand All @@ -1453,7 +1453,7 @@ pub static MZ_CONNECTIONS: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
desc: RelationDesc::empty()
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("schema_id", ScalarType::UInt64.nullable(false))
.with_column("schema_id", ScalarType::String.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column("type", ScalarType::String.nullable(false))
.with_column("owner_id", ScalarType::String.nullable(false)),
Expand All @@ -1474,7 +1474,7 @@ pub static MZ_SOURCES: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
desc: RelationDesc::empty()
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("schema_id", ScalarType::UInt64.nullable(false))
.with_column("schema_id", ScalarType::String.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column("type", ScalarType::String.nullable(false))
.with_column("connection_id", ScalarType::String.nullable(true))
Expand All @@ -1490,7 +1490,7 @@ pub static MZ_SINKS: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
desc: RelationDesc::empty()
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("schema_id", ScalarType::UInt64.nullable(false))
.with_column("schema_id", ScalarType::String.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column("type", ScalarType::String.nullable(false))
.with_column("connection_id", ScalarType::String.nullable(true))
Expand All @@ -1506,7 +1506,7 @@ pub static MZ_VIEWS: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
desc: RelationDesc::empty()
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("schema_id", ScalarType::UInt64.nullable(false))
.with_column("schema_id", ScalarType::String.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column("definition", ScalarType::String.nullable(false))
.with_column("owner_id", ScalarType::String.nullable(false)),
Expand All @@ -1518,7 +1518,7 @@ pub static MZ_MATERIALIZED_VIEWS: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable
desc: RelationDesc::empty()
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("schema_id", ScalarType::UInt64.nullable(false))
.with_column("schema_id", ScalarType::String.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column("cluster_id", ScalarType::String.nullable(false))
.with_column("definition", ScalarType::String.nullable(false))
Expand All @@ -1531,7 +1531,7 @@ pub static MZ_TYPES: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
desc: RelationDesc::empty()
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("schema_id", ScalarType::UInt64.nullable(false))
.with_column("schema_id", ScalarType::String.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column("category", ScalarType::String.nullable(false))
.with_column("owner_id", ScalarType::String.nullable(false)),
Expand Down Expand Up @@ -1602,7 +1602,7 @@ pub static MZ_FUNCTIONS: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
desc: RelationDesc::empty()
.with_column("id", ScalarType::String.nullable(false))
.with_column("oid", ScalarType::Oid.nullable(false))
.with_column("schema_id", ScalarType::UInt64.nullable(false))
.with_column("schema_id", ScalarType::String.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column(
"argument_type_ids",
Expand Down Expand Up @@ -1655,7 +1655,7 @@ pub static MZ_SECRETS: Lazy<BuiltinTable> = Lazy::new(|| BuiltinTable {
schema: MZ_CATALOG_SCHEMA,
desc: RelationDesc::empty()
.with_column("id", ScalarType::String.nullable(false))
.with_column("schema_id", ScalarType::UInt64.nullable(false))
.with_column("schema_id", ScalarType::String.nullable(false))
.with_column("name", ScalarType::String.nullable(false))
.with_column("owner_id", ScalarType::String.nullable(false)),
is_retained_metrics_object: false,
Expand Down
26 changes: 13 additions & 13 deletions src/adapter/src/catalog/builtin_table_updates.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ impl CatalogState {
BuiltinTableUpdate {
id: self.resolve_builtin_table(&MZ_DATABASES),
row: Row::pack_slice(&[
Datum::UInt64(database.id.0),
Datum::String(&database.id.to_string()),
Datum::UInt32(database.oid),
Datum::String(database.name()),
Datum::String(&database.owner_id.to_string()),
Expand All @@ -109,16 +109,16 @@ impl CatalogState {
let (database_id, schema) = match database_spec {
ResolvedDatabaseSpecifier::Ambient => (None, &self.ambient_schemas_by_id[schema_id]),
ResolvedDatabaseSpecifier::Id(id) => (
Some(id.0),
Some(id.to_string()),
&self.database_by_id[id].schemas_by_id[schema_id],
),
};
BuiltinTableUpdate {
id: self.resolve_builtin_table(&MZ_SCHEMAS),
row: Row::pack_slice(&[
Datum::UInt64(schema_id.0),
Datum::String(&schema_id.to_string()),
Datum::UInt32(schema.oid),
Datum::from(database_id),
Datum::from(database_id.as_deref()),
Datum::String(&schema.name.schema),
Datum::String(&schema.owner_id.to_string()),
]),
Expand Down Expand Up @@ -397,7 +397,7 @@ impl CatalogState {
row: Row::pack_slice(&[
Datum::String(&id.to_string()),
Datum::UInt32(oid),
Datum::UInt64(schema_id.into()),
Datum::String(&schema_id.to_string()),
Datum::String(name),
Datum::String(&owner_id.to_string()),
]),
Expand All @@ -424,7 +424,7 @@ impl CatalogState {
row: Row::pack_slice(&[
Datum::String(&id.to_string()),
Datum::UInt32(oid),
Datum::UInt64(schema_id.into()),
Datum::String(&schema_id.to_string()),
Datum::String(name),
Datum::String(source_desc_name),
Datum::from(connection_id.map(|id| id.to_string()).as_deref()),
Expand Down Expand Up @@ -468,7 +468,7 @@ impl CatalogState {
row: Row::pack_slice(&[
Datum::String(&id.to_string()),
Datum::UInt32(oid),
Datum::UInt64(schema_id.into()),
Datum::String(&schema_id.to_string()),
Datum::String(name),
Datum::String(match connection.connection {
mz_storage_client::types::connections::Connection::Kafka { .. } => "kafka",
Expand Down Expand Up @@ -615,7 +615,7 @@ impl CatalogState {
row: Row::pack_slice(&[
Datum::String(&id.to_string()),
Datum::UInt32(oid),
Datum::UInt64(schema_id.into()),
Datum::String(&schema_id.to_string()),
Datum::String(name),
Datum::String(&query_string),
Datum::String(&owner_id.to_string()),
Expand Down Expand Up @@ -652,7 +652,7 @@ impl CatalogState {
row: Row::pack_slice(&[
Datum::String(&id.to_string()),
Datum::UInt32(oid),
Datum::UInt64(schema_id.into()),
Datum::String(&schema_id.to_string()),
Datum::String(name),
Datum::String(&mview.cluster_id.to_string()),
Datum::String(&query_string),
Expand Down Expand Up @@ -698,7 +698,7 @@ impl CatalogState {
row: Row::pack_slice(&[
Datum::String(&id.to_string()),
Datum::UInt32(oid),
Datum::UInt64(schema_id.into()),
Datum::String(&schema_id.to_string()),
Datum::String(name),
Datum::String(connection.name()),
Datum::from(sink.connection_id().map(|id| id.to_string()).as_deref()),
Expand Down Expand Up @@ -800,7 +800,7 @@ impl CatalogState {
row: Row::pack_slice(&[
Datum::String(&id.to_string()),
Datum::UInt32(oid),
Datum::UInt64(schema_id.into()),
Datum::String(&schema_id.to_string()),
Datum::String(name),
Datum::String(&TypeCategory::from_catalog_type(&typ.details.typ).to_string()),
Datum::String(&owner_id.to_string()),
Expand Down Expand Up @@ -882,7 +882,7 @@ impl CatalogState {
row: Row::pack_slice(&[
Datum::String(&id.to_string()),
Datum::UInt32(func_impl_details.oid),
Datum::UInt64(schema_id.into()),
Datum::String(&schema_id.to_string()),
Datum::String(name),
arg_type_ids,
Datum::from(
Expand Down Expand Up @@ -959,7 +959,7 @@ impl CatalogState {
id: self.resolve_builtin_table(&MZ_SECRETS),
row: Row::pack_slice(&[
Datum::String(&id.to_string()),
Datum::UInt64(schema_id.into()),
Datum::String(&schema_id.to_string()),
Datum::String(name),
Datum::String(&owner_id.to_string()),
]),
Expand Down