From fcbff8fa19a5546e058d1de0b9a9475c5d5e6fe2 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 12 Nov 2021 13:55:01 +0300 Subject: [PATCH 01/31] Correct type from Alias::new("ploygon") to Alias::new("polygon") Add enum type for writer --- src/postgres/writer/column.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/postgres/writer/column.rs b/src/postgres/writer/column.rs index 8b46dd55..0231c3e7 100644 --- a/src/postgres/writer/column.rs +++ b/src/postgres/writer/column.rs @@ -156,7 +156,7 @@ impl ColumnInfo { col_def.custom(Alias::new("path")); } Type::Polygon => { - col_def.custom(Alias::new("ploygon")); + col_def.custom(Alias::new("polygon")); } Type::Circle => { col_def.custom(Alias::new("circle")); @@ -227,6 +227,9 @@ impl ColumnInfo { Type::Unknown(s) => { col_def.custom(Alias::new(s)); } + Type::Enum(def) => { + col_def.custom(Alias::new("enum")); + } }; col_def } From a2b063a35b7fa0344f381fe487b9f0d049f25c62 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 12 Nov 2021 13:55:52 +0300 Subject: [PATCH 02/31] Add type --- src/postgres/def/types.rs | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index ae5a0c04..62e2377b 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -138,6 +138,7 @@ pub enum Type { PgLsn, // TODO: Pseudo-types Unknown(String), + Enum(EnumDef), } impl Type { @@ -194,6 +195,7 @@ impl Type { "daterange" => Type::DateRange, // "" => Type::Domain, "pg_lsn" => Type::PgLsn, + "enum" => Type::Enum(EnumDef::default()), _ => Type::Unknown(name.to_owned()), } @@ -263,4 +265,47 @@ impl Type { pub fn has_bit_attr(&self) -> bool { matches!(self, Type::Bit(_)) } + + pub fn get_enum_def(&self) -> &EnumDef { + match self { + Type::Enum(def) => def, + _ => panic!("type error"), + } + } + + pub fn get_enum_def_mut(&mut self) -> &mut EnumDef { + match self { + Type::Enum(def) => def, + _ => panic!("type error"), + } + } + + pub fn is_enum(&self) -> bool { + matches!(self, Type::Enum(_)) + } +} + +#[derive(Debug)] +pub struct EnumIden(String); + +impl sea_query::Iden for EnumIden { + fn unquoted(&self, s: &mut dyn std::fmt::Write) { + write!(s, "{}", self.0).unwrap(); + } +} + +#[derive(Clone, Debug, Default, PartialEq)] +#[cfg_attr(feature = "with-serde", derive(Serialize, Deserialize))] +pub struct EnumDef { + pub values: Vec, + pub attr: StringAttr, +} + +impl EnumDef { + pub fn to_enum_def(&self) -> Vec { + self.values + .iter() + .map(|iden| EnumIden(iden.to_owned())) + .collect::>() + } } From 57e5a4176e06a6c49bee8dedbf9977ce5156410d Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Mon, 15 Nov 2021 10:40:38 +0300 Subject: [PATCH 03/31] Add support for PostgreSQL types Fix a few typos I found while digging through the source code --- src/postgres/def/types.rs | 133 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 129 insertions(+), 4 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 62e2377b..b62a39d4 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -1,3 +1,8 @@ +use crate::parser::Parser; +use sea_query::{ + backend::PostgresQueryBuilder, extension::postgres::TypeCreateStatement, unescape_string, Token, +}; + #[cfg(feature = "with-serde")] use serde::{Deserialize, Serialize}; @@ -91,7 +96,7 @@ pub enum Type { Bit(BitAttr), // Text search types - /// A sorted list of distincp lexemes which are words that have been normalized to merge different + /// A sorted list of distinct lexemes which are words that have been normalized to merge different /// variants of the same word TsVector, /// A list of lexemes that are to be searched for, and can be combined using Boolean operators AND, @@ -101,7 +106,7 @@ pub enum Type { /// A universally unique identifier as defined by RFC 4122, ISO 9834-8:2005, and related standards Uuid, - /// XML data checked for well-formedness and with additonal support functions + /// XML data checked for well-formedness and with additional support functions Xml, /// JSON data checked for validity and with additional functions @@ -138,6 +143,7 @@ pub enum Type { PgLsn, // TODO: Pseudo-types Unknown(String), + /// Defines an PostgreSQL Enum(EnumDef), } @@ -200,6 +206,56 @@ impl Type { _ => Type::Unknown(name.to_owned()), } } + + /// Parses a raw SQL enum `CREATE TYPE name AS ENUM(field_one, field_two, ...)` + pub fn enum_from_query(query: &str) -> Type { + let mut parser = Parser::new(&query); + + let mut column_type = Type::Enum(EnumDef::default()); + + let mut enum_name = String::default(); + + while let Some(_) = parser.next() { + if parser.last == Some(Token::Unquoted("TYPE".to_owned())) && enum_name.is_empty() { + match parser.curr() { + None => (), + Some(token) => match token { + Token::Quoted(_) => { + if let Some(unquoted) = token.unquote() { + enum_name = unquoted; + } + } + Token::Unquoted(unquoted_token) => enum_name = unquoted_token.to_owned(), + _ => (), + }, + }; + } + + if parser.next_if_punctuation("(") { + while parser.curr().is_some() { + if let Some(word) = parser.next_if_quoted_any() { + column_type + .get_enum_def_mut() + .values + .push(unescape_string(word.unquote().unwrap().as_str())); + parser.next_if_punctuation(","); + } else if parser.curr_is_unquoted() { + todo!("there can actually be numeric enum values but is very confusing"); + } + if parser.next_if_punctuation(")") { + break; + } + } + } + } + + let attr_len = enum_name.len() as u16; + + column_type.get_enum_def_mut().typename = enum_name; + column_type.get_enum_def_mut().attr.length = Some(attr_len); + + column_type + } } #[derive(Clone, Debug, PartialEq, Default)] @@ -283,10 +339,19 @@ impl Type { pub fn is_enum(&self) -> bool { matches!(self, Type::Enum(_)) } + + /// Returns nome if the type is not an enum `Type::Enum(EnumDef)` + pub fn enum_to_create_statement(&self) -> Option { + match self { + Type::Enum(enum_def) => Some(enum_def.to_sql_query()), + _ => None, + } + } } +/// Used to ensure enum names and enum fields always implement [sea_query::types::IntoIden] #[derive(Debug)] -pub struct EnumIden(String); +pub struct EnumIden(pub String); impl sea_query::Iden for EnumIden { fn unquoted(&self, s: &mut dyn std::fmt::Write) { @@ -294,18 +359,78 @@ impl sea_query::Iden for EnumIden { } } +/// Defines an enum for the PostgreSQL module #[derive(Clone, Debug, Default, PartialEq)] #[cfg_attr(feature = "with-serde", derive(Serialize, Deserialize))] pub struct EnumDef { + /// Holds the fields of the `ENUM` pub values: Vec, + /// Defines the length of the name describing the [Type::Enum] pub attr: StringAttr, + /// Defines the name of the PostgreSQL enum identifier + pub typename: String, } impl EnumDef { - pub fn to_enum_def(&self) -> Vec { + /// Implements [sea_query::types::IntoIden] for the Enum name + pub fn typename_impl_iden(&self) -> EnumIden { + EnumIden(self.typename.to_owned()) + } + + /// Implements [sea_query::types::IntoIden] for the Enum fields + pub fn values_impl_iden(&self) -> Vec { self.values .iter() .map(|iden| EnumIden(iden.to_owned())) .collect::>() } + + /// Converts the [EnumDef] to a [TypeCreateStatement] + pub fn to_create_statement(&self) -> TypeCreateStatement { + sea_query::extension::postgres::Type::create() + .as_enum(self.typename_impl_iden()) + .values(self.values_impl_iden()) + .clone() + } + + /// Converts the [EnumDef] to a SQL statement + pub fn to_sql_query(&self) -> String { + sea_query::extension::postgres::Type::create() + .as_enum(self.typename_impl_iden()) + .values(self.values_impl_iden()) + .to_string(PostgresQueryBuilder) + } +} + +/// Helpers to extract the Postgres `ENUM` type from a PostgreSQL row +#[derive(sqlx::FromRow, Debug)] +pub struct PgEnum { + /// The name of the enum type + pub typname: String, + /// A field of the enum type. + pub enumlabel: String, +} + +impl PgEnum { + /// Create a new type to handle a Postgres ENUM row + pub fn new() -> Self { + Self { + typname: String::default(), + enumlabel: String::default(), + } + } + + /// Create a [Type::Enum] from a PostgreSQL enum row + pub fn to_enum_type(&mut self, fields: Vec) -> Type { + let mut enum_def = EnumDef::default(); + let typename = fields[0].typname.clone(); + enum_def.attr.length = Some(typename.len() as u16); + enum_def.typename = typename; + + fields.iter().for_each(|pg_enum| { + enum_def.values.push(pg_enum.enumlabel.clone()); + }); + + Type::Enum(enum_def) + } } From 7e36665b81e7e8d59c0d073cdd8aa7e6878f92c9 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Mon, 15 Nov 2021 10:41:58 +0300 Subject: [PATCH 04/31] Add tests for checking the correctness of PostgreSQL enum parsing and processing --- Cargo.toml | 1 + tests/writer/postgres_enum/Cargo.toml | 12 ++++ tests/writer/postgres_enum/Readme.md | 5 ++ tests/writer/postgres_enum/src/main.rs | 90 ++++++++++++++++++++++++++ 4 files changed, 108 insertions(+) create mode 100644 tests/writer/postgres_enum/Cargo.toml create mode 100644 tests/writer/postgres_enum/Readme.md create mode 100644 tests/writer/postgres_enum/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 6759a5e0..12cf0761 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "tests/discovery/postgres", "tests/writer/mysql", "tests/writer/postgres", + "tests/writer/postgres_enum", "tests/live/mysql", "tests/live/postgres", ] diff --git a/tests/writer/postgres_enum/Cargo.toml b/tests/writer/postgres_enum/Cargo.toml new file mode 100644 index 00000000..8e10c841 --- /dev/null +++ b/tests/writer/postgres_enum/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "sea-schema-writer-test-postgres-enum" +version = "0.1.0" +edition = "2018" +publish = false + +[dependencies] +async-std = { version = "1.8", features = [ "attributes" ] } +sea-query = "0.18.2" +sea-schema = { path = "../../../", default-features = false, features = [ "sqlx-postgres", "runtime-async-std-native-tls", "discovery", "writer", "debug-print" ] } +serde_json = { version = "^1" } +sqlx = { version = "^0" } diff --git a/tests/writer/postgres_enum/Readme.md b/tests/writer/postgres_enum/Readme.md new file mode 100644 index 00000000..6946a346 --- /dev/null +++ b/tests/writer/postgres_enum/Readme.md @@ -0,0 +1,5 @@ +# Run + +```sh +cargo run > schema.sql +``` \ No newline at end of file diff --git a/tests/writer/postgres_enum/src/main.rs b/tests/writer/postgres_enum/src/main.rs new file mode 100644 index 00000000..978339ba --- /dev/null +++ b/tests/writer/postgres_enum/src/main.rs @@ -0,0 +1,90 @@ +use sea_schema::postgres::def::{EnumDef, PgEnum, StringAttr, Type}; +use sqlx::PgPool; + +#[async_std::main] +async fn main() { + let connection = PgPool::connect("postgres://sea:sea@localhost/sakila") + .await + .unwrap(); + + let rows: Vec = sqlx::query_as("SELECT pg_type.typname, pg_enum.enumlabel FROM pg_type JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid;", + ) + .fetch_all(&connection) + .await + .unwrap(); + + let custom_enum = PgEnum::new().to_enum_type(rows); + + let font_family = r#"CREATE TYPE "font_family" AS ENUM ('serif', 'sans', 'monospace')"#; + + let rating = r#"CREATE TYPE "mpaa_rating" AS ENUM ('G', 'PG', 'PG-13', 'R', 'NC-17')"#; + + let parse_font_family = Type::enum_from_query(font_family); + let parse_rating = Type::enum_from_query(rating); + + assert_eq!( + parse_font_family, + Type::Enum(EnumDef { + values: vec![ + "serif".to_owned(), + "sans".to_owned(), + "monospace".to_owned(), + ], + attr: StringAttr { length: Some(11,) }, + typename: "font_family".to_owned(), + },) + ); + + // Check if the end of parsed statement is same as the sql query + assert_eq!( + match parse_font_family { + Type::Enum(enum_def) => Some(enum_def.to_sql_query()), + _ => None, + }, + Some(font_family.to_owned()) + ); + + assert_eq!( + parse_rating, + Type::Enum(EnumDef { + values: vec![ + "G".to_owned(), + "PG".to_owned(), + "PG-13".to_owned(), + "R".to_owned(), + "NC-17".to_owned(), + ], + attr: StringAttr { length: Some(11,) }, + typename: "mpaa_rating".to_owned(), + },) + ); + + // Check if the end of parsed statement is same as the sql query + assert_eq!( + match parse_rating { + Type::Enum(enum_def) => Some(enum_def.to_sql_query()), + _ => None, + }, + Some(rating.to_owned()) + ); + + assert_eq!( + custom_enum.enum_to_create_statement(), + Some(r#"CREATE TYPE "mpaa_rating" AS ENUM ('G', 'PG', 'PG-13', 'R', 'NC-17')"#.to_owned()) + ); + + assert_eq!( + custom_enum, + Type::Enum(EnumDef { + values: vec![ + "G".to_owned(), + "PG".to_owned(), + "PG-13".to_owned(), + "R".to_owned(), + "NC-17".to_owned(), + ], + attr: StringAttr { length: Some(11,) }, + typename: "mpaa_rating".to_owned(), + },) + ); +} From d2016a11816819cf5fff15ed7b35f560f56be79c Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Mon, 15 Nov 2021 11:03:54 +0300 Subject: [PATCH 05/31] Gate `PgEnum` and its impl block under the `sqlx` feature to allow the ` #[derive(sqlx::FromRow, Debug)]` to be used to derive an sqlx row for handling PostgreSQL enums --- src/postgres/def/types.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index b62a39d4..01b1c497 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -402,6 +402,7 @@ impl EnumDef { } } +#[cfg(feature = "sqlx")] /// Helpers to extract the Postgres `ENUM` type from a PostgreSQL row #[derive(sqlx::FromRow, Debug)] pub struct PgEnum { @@ -411,6 +412,7 @@ pub struct PgEnum { pub enumlabel: String, } +#[cfg(feature = "sqlx")] impl PgEnum { /// Create a new type to handle a Postgres ENUM row pub fn new() -> Self { From d70fe565a9bed98afa073339f1881075bb3a2645 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Thu, 18 Nov 2021 16:40:13 +0300 Subject: [PATCH 06/31] Add support for fetching enums --- src/postgres/discovery/executor/real.rs | 38 ++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/postgres/discovery/executor/real.rs b/src/postgres/discovery/executor/real.rs index 84cc35d4..249e692a 100644 --- a/src/postgres/discovery/executor/real.rs +++ b/src/postgres/discovery/executor/real.rs @@ -1,5 +1,5 @@ use sea_query::{PostgresQueryBuilder, SelectStatement}; -use sqlx::{postgres::PgRow, PgPool}; +use sqlx::{postgres::PgRow, PgPool, Row}; sea_query::sea_query_driver_postgres!(); use sea_query_driver_postgres::bind_query; @@ -31,4 +31,40 @@ impl Executor { .await .unwrap() } + + pub async fn get_enums(&self, select: SelectStatement) -> Vec { + let (sql, values) = select.build(PostgresQueryBuilder); + debug_print!("{}, {:?}", sql, values); + + let query = bind_query(sqlx::query(&sql), &values); + + let rows = query + .fetch_all(&mut self.pool.acquire().await.unwrap()) + .await + .unwrap(); + + rows.iter() + .map(|pg_row| { + let column: EnumColumn = pg_row.into(); + + column + }) + .collect::>() + } +} + +#[derive(Debug, sqlx::FromRow)] +pub struct EnumColumn { + pub name: String, + pub value: String, +} + +// #[cfg(feature = "sqlx-postgres")] +impl From<&PgRow> for EnumColumn { + fn from(row: &PgRow) -> Self { + Self { + name: row.get(0), + value: row.get(1), + } + } } From b57b8ddadc89959a1f0b1f24203458f83ce692a0 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Thu, 18 Nov 2021 17:52:50 +0300 Subject: [PATCH 07/31] Move types into types files under postgres::def::Type Add documentation on types Output the result by calling `discover_enum` on `SchemaDiscovery` --- src/postgres/def/types.rs | 109 +++++++----------------- src/postgres/discovery/executor/real.rs | 28 ++---- src/postgres/discovery/mod.rs | 44 +++++++++- 3 files changed, 81 insertions(+), 100 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 01b1c497..c95adf02 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -1,7 +1,5 @@ -use crate::parser::Parser; -use sea_query::{ - backend::PostgresQueryBuilder, extension::postgres::TypeCreateStatement, unescape_string, Token, -}; +use sea_query::{backend::PostgresQueryBuilder, extension::postgres::TypeCreateStatement}; +use sqlx::Row; #[cfg(feature = "with-serde")] use serde::{Deserialize, Serialize}; @@ -206,56 +204,6 @@ impl Type { _ => Type::Unknown(name.to_owned()), } } - - /// Parses a raw SQL enum `CREATE TYPE name AS ENUM(field_one, field_two, ...)` - pub fn enum_from_query(query: &str) -> Type { - let mut parser = Parser::new(&query); - - let mut column_type = Type::Enum(EnumDef::default()); - - let mut enum_name = String::default(); - - while let Some(_) = parser.next() { - if parser.last == Some(Token::Unquoted("TYPE".to_owned())) && enum_name.is_empty() { - match parser.curr() { - None => (), - Some(token) => match token { - Token::Quoted(_) => { - if let Some(unquoted) = token.unquote() { - enum_name = unquoted; - } - } - Token::Unquoted(unquoted_token) => enum_name = unquoted_token.to_owned(), - _ => (), - }, - }; - } - - if parser.next_if_punctuation("(") { - while parser.curr().is_some() { - if let Some(word) = parser.next_if_quoted_any() { - column_type - .get_enum_def_mut() - .values - .push(unescape_string(word.unquote().unwrap().as_str())); - parser.next_if_punctuation(","); - } else if parser.curr_is_unquoted() { - todo!("there can actually be numeric enum values but is very confusing"); - } - if parser.next_if_punctuation(")") { - break; - } - } - } - } - - let attr_len = enum_name.len() as u16; - - column_type.get_enum_def_mut().typename = enum_name; - column_type.get_enum_def_mut().attr.length = Some(attr_len); - - column_type - } } #[derive(Clone, Debug, PartialEq, Default)] @@ -402,37 +350,40 @@ impl EnumDef { } } -#[cfg(feature = "sqlx")] -/// Helpers to extract the Postgres `ENUM` type from a PostgreSQL row -#[derive(sqlx::FromRow, Debug)] -pub struct PgEnum { - /// The name of the enum type - pub typname: String, - /// A field of the enum type. - pub enumlabel: String, +/// Holds the enum and their values from a `PgRow` +#[derive(Debug, sqlx::FromRow)] +pub struct EnumRow { + // The name of the enum type + pub name: String, + /// The values of the enum type which are concatenated using ` | ` symbol + /// for example `"sans|serif|monospace"` + pub values: String, } -#[cfg(feature = "sqlx")] -impl PgEnum { - /// Create a new type to handle a Postgres ENUM row - pub fn new() -> Self { +use crate::sqlx_types::postgres::PgRow; +impl From<&PgRow> for EnumRow { + fn from(row: &PgRow) -> Self { Self { - typname: String::default(), - enumlabel: String::default(), + name: row.get(0), + values: row.get(1), } } +} - /// Create a [Type::Enum] from a PostgreSQL enum row - pub fn to_enum_type(&mut self, fields: Vec) -> Type { - let mut enum_def = EnumDef::default(); - let typename = fields[0].typname.clone(); - enum_def.attr.length = Some(typename.len() as u16); - enum_def.typename = typename; - - fields.iter().for_each(|pg_enum| { - enum_def.values.push(pg_enum.enumlabel.clone()); - }); +impl From<&EnumRow> for EnumDef { + fn from(row: &EnumRow) -> Self { + let fields = row + .values + .split("|") + .map(|field| field.to_owned()) + .collect::>(); - Type::Enum(enum_def) + Self { + typename: row.name.to_owned(), + attr: StringAttr { + length: Some(row.name.len() as u16), + }, + values: fields, + } } } diff --git a/src/postgres/discovery/executor/real.rs b/src/postgres/discovery/executor/real.rs index 249e692a..0df35ead 100644 --- a/src/postgres/discovery/executor/real.rs +++ b/src/postgres/discovery/executor/real.rs @@ -1,5 +1,6 @@ +use crate::postgres::def::EnumRow; use sea_query::{PostgresQueryBuilder, SelectStatement}; -use sqlx::{postgres::PgRow, PgPool, Row}; +use sqlx::{postgres::PgRow, PgPool}; sea_query::sea_query_driver_postgres!(); use sea_query_driver_postgres::bind_query; @@ -32,7 +33,10 @@ impl Executor { .unwrap() } - pub async fn get_enums(&self, select: SelectStatement) -> Vec { + /// Fetches enums from the enum column. There are many ways to do this however, + /// this function uses the SQL statement + /// `SELECT type.typname AS name, string_agg(enum.enumlabel, '|') AS value FROM pg_enum AS enum JOIN pg_type AS type ON type.oid = enum.enumtypid GROUP BY type.typname; ` + pub async fn get_enums(&self, select: SelectStatement) -> Vec { let (sql, values) = select.build(PostgresQueryBuilder); debug_print!("{}, {:?}", sql, values); @@ -45,26 +49,10 @@ impl Executor { rows.iter() .map(|pg_row| { - let column: EnumColumn = pg_row.into(); + let column: EnumRow = pg_row.into(); column }) - .collect::>() - } -} - -#[derive(Debug, sqlx::FromRow)] -pub struct EnumColumn { - pub name: String, - pub value: String, -} - -// #[cfg(feature = "sqlx-postgres")] -impl From<&PgRow> for EnumColumn { - fn from(row: &PgRow) -> Self { - Self { - name: row.get(0), - value: row.get(1), - } + .collect::>() } } diff --git a/src/postgres/discovery/mod.rs b/src/postgres/discovery/mod.rs index e64d643c..67724f18 100644 --- a/src/postgres/discovery/mod.rs +++ b/src/postgres/discovery/mod.rs @@ -7,7 +7,7 @@ use crate::postgres::query::{ ColumnQueryResult, SchemaQueryBuilder, TableConstraintsQueryResult, TableQueryResult, }; use futures::future; -use sea_query::{Alias, Iden, IntoIden, SeaRc}; +use sea_query::{Alias, ColumnRef, Expr, Iden, IntoIden, JoinType, SeaRc, SelectStatement}; mod executor; pub use executor::*; @@ -176,4 +176,46 @@ impl SchemaDiscovery { constraints } + + pub async fn discover_enums(&self) -> Vec { + let mut query = SelectStatement::new(); + + // SELECT type.typname AS name, string_agg(enum.enumlabel, '|') AS value FROM pg_enum AS enum JOIN pg_type AS type ON type.oid = enum.enumtypid GROUP BY type.typname + query + .expr_as( + Expr::col(ColumnRef::TableColumn( + SeaRc::new(Alias::new("type")), + SeaRc::new(Alias::new("typname")), + )), + Alias::new("name"), + ) + .expr_as( + Expr::cust("string_agg(enum.enumlabel, '|')"), + Alias::new("value"), + ) + .from_as(Alias::new("pg_enum"), Alias::new("enum")) + .join_as( + JoinType::Join, + Alias::new("pg_type"), + Alias::new("type"), + Expr::tbl(Alias::new("type"), Alias::new("oid")) + .equals(Alias::new("enum"), Alias::new("enumtypid")), + ) + .group_by_col(ColumnRef::TableColumn( + SeaRc::new(Alias::new("type")), + SeaRc::new(Alias::new("typname")), + )); + + let enum_rows = self.executor.get_enums(query).await; + + let mut enums: Vec = Vec::default(); + + enum_rows.iter().for_each(|enum_row| { + let enum_def: EnumDef = enum_row.into(); + + enums.push(Type::Enum(enum_def)); + }); + + enums + } } From a8d9b4dca65ffc44c6ff0d5a24060926089faf85 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 19 Nov 2021 09:46:21 +0300 Subject: [PATCH 08/31] Add method to parse a raw SQL create query for enum creation Add missing documentation for the added types for enum creation --- src/postgres/def/types.rs | 56 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index c95adf02..bda9d142 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -269,7 +269,7 @@ impl Type { pub fn has_bit_attr(&self) -> bool { matches!(self, Type::Bit(_)) } - + /// Get an immutable reference to the [EnumDef] of type [Type::Enum] pub fn get_enum_def(&self) -> &EnumDef { match self { Type::Enum(def) => def, @@ -277,6 +277,7 @@ impl Type { } } + /// Get a mutable reference to the [EnumDef] of type [Type::Enum] pub fn get_enum_def_mut(&mut self) -> &mut EnumDef { match self { Type::Enum(def) => def, @@ -284,6 +285,7 @@ impl Type { } } + /// Is the type given an enum pub fn is_enum(&self) -> bool { matches!(self, Type::Enum(_)) } @@ -295,6 +297,58 @@ impl Type { _ => None, } } + /// Convert a raw SQL query into a type [Type::Enum] + pub fn enum_from_query(query: &str) -> Type { + use crate::parser::Parser; + use sea_query::{unescape_string, Token}; + + let mut parser = Parser::new(&query); + + let mut type_enum = Type::Enum(EnumDef::default()); + + let mut enum_name = String::default(); + + while let Some(_) = parser.next() { + if parser.last == Some(Token::Unquoted("TYPE".to_owned())) && enum_name.is_empty() { + match parser.curr() { + None => (), + Some(token) => match token { + Token::Quoted(_) => { + if let Some(unquoted) = token.unquote() { + enum_name = unquoted; + } + } + Token::Unquoted(unquoted_token) => enum_name = unquoted_token.to_owned(), + _ => (), + }, + }; + } + + if parser.next_if_punctuation("(") { + while parser.curr().is_some() { + if let Some(word) = parser.next_if_quoted_any() { + type_enum + .get_enum_def_mut() + .values + .push(unescape_string(word.unquote().unwrap().as_str())); + parser.next_if_punctuation(","); + } else if parser.curr_is_unquoted() { + todo!("there can actually be numeric enum values but is very confusing"); + } + if parser.next_if_punctuation(")") { + break; + } + } + } + } + + let attr_len = enum_name.len() as u16; + + type_enum.get_enum_def_mut().typename = enum_name; + type_enum.get_enum_def_mut().attr.length = Some(attr_len); + + type_enum + } } /// Used to ensure enum names and enum fields always implement [sea_query::types::IntoIden] From beabb80ae330d0443730fca54826d47694b2f251 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 19 Nov 2021 09:48:20 +0300 Subject: [PATCH 09/31] Test case to discover all the enums in a given database schema Write into SQL and then parse the raw SQL asserting that all discovered enums are the same as their raw SQL queries --- tests/writer/postgres_enum/src/main.rs | 90 +++----------------------- 1 file changed, 10 insertions(+), 80 deletions(-) diff --git a/tests/writer/postgres_enum/src/main.rs b/tests/writer/postgres_enum/src/main.rs index 978339ba..eee689e6 100644 --- a/tests/writer/postgres_enum/src/main.rs +++ b/tests/writer/postgres_enum/src/main.rs @@ -1,4 +1,4 @@ -use sea_schema::postgres::def::{EnumDef, PgEnum, StringAttr, Type}; +use sea_schema::postgres::{def::Type, discovery::SchemaDiscovery}; use sqlx::PgPool; #[async_std::main] @@ -7,84 +7,14 @@ async fn main() { .await .unwrap(); - let rows: Vec = sqlx::query_as("SELECT pg_type.typname, pg_enum.enumlabel FROM pg_type JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid;", - ) - .fetch_all(&connection) - .await - .unwrap(); + let enums_discovery = SchemaDiscovery::new(connection, "public") + .discover_enums() + .await; - let custom_enum = PgEnum::new().to_enum_type(rows); - - let font_family = r#"CREATE TYPE "font_family" AS ENUM ('serif', 'sans', 'monospace')"#; - - let rating = r#"CREATE TYPE "mpaa_rating" AS ENUM ('G', 'PG', 'PG-13', 'R', 'NC-17')"#; - - let parse_font_family = Type::enum_from_query(font_family); - let parse_rating = Type::enum_from_query(rating); - - assert_eq!( - parse_font_family, - Type::Enum(EnumDef { - values: vec![ - "serif".to_owned(), - "sans".to_owned(), - "monospace".to_owned(), - ], - attr: StringAttr { length: Some(11,) }, - typename: "font_family".to_owned(), - },) - ); - - // Check if the end of parsed statement is same as the sql query - assert_eq!( - match parse_font_family { - Type::Enum(enum_def) => Some(enum_def.to_sql_query()), - _ => None, - }, - Some(font_family.to_owned()) - ); - - assert_eq!( - parse_rating, - Type::Enum(EnumDef { - values: vec![ - "G".to_owned(), - "PG".to_owned(), - "PG-13".to_owned(), - "R".to_owned(), - "NC-17".to_owned(), - ], - attr: StringAttr { length: Some(11,) }, - typename: "mpaa_rating".to_owned(), - },) - ); - - // Check if the end of parsed statement is same as the sql query - assert_eq!( - match parse_rating { - Type::Enum(enum_def) => Some(enum_def.to_sql_query()), - _ => None, - }, - Some(rating.to_owned()) - ); - - assert_eq!( - custom_enum.enum_to_create_statement(), - Some(r#"CREATE TYPE "mpaa_rating" AS ENUM ('G', 'PG', 'PG-13', 'R', 'NC-17')"#.to_owned()) - ); - - assert_eq!( - custom_enum, - Type::Enum(EnumDef { - values: vec![ - "G".to_owned(), - "PG".to_owned(), - "PG-13".to_owned(), - "R".to_owned(), - "NC-17".to_owned(), - ], - attr: StringAttr { length: Some(11,) }, - typename: "mpaa_rating".to_owned(), - },) - ); + // Assert that all enums in the database are equal to their + // raw SQL queries + enums_discovery.iter().for_each(|enum_type| { + let to_sql_query = enum_type.get_enum_def().to_sql_query(); + assert_eq!(enum_type, &Type::enum_from_query(&to_sql_query)); + }); } From 8be474166d296e316abc29b13d9a89a5714c1721 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 19 Nov 2021 10:09:22 +0300 Subject: [PATCH 10/31] Solve issues with github actions not detecting types --- src/postgres/def/types.rs | 3 +++ src/postgres/discovery/mod.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index bda9d142..080c0e72 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -1,4 +1,6 @@ use sea_query::{backend::PostgresQueryBuilder, extension::postgres::TypeCreateStatement}; + +#[cfg(feature = "sqlx-dep")] use sqlx::Row; #[cfg(feature = "with-serde")] @@ -405,6 +407,7 @@ impl EnumDef { } /// Holds the enum and their values from a `PgRow` +#[cfg(feature = "sqlx-dep")] #[derive(Debug, sqlx::FromRow)] pub struct EnumRow { // The name of the enum type diff --git a/src/postgres/discovery/mod.rs b/src/postgres/discovery/mod.rs index 67724f18..546ff73f 100644 --- a/src/postgres/discovery/mod.rs +++ b/src/postgres/discovery/mod.rs @@ -177,6 +177,7 @@ impl SchemaDiscovery { constraints } + #[cfg(feature = "sqlx-postgres")] pub async fn discover_enums(&self) -> Vec { let mut query = SelectStatement::new(); From ec076a70ff6e5aede6b32a31dcc716b24b36e068 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 19 Nov 2021 10:21:34 +0300 Subject: [PATCH 11/31] Add feature flag for new enum type and associated functions --- src/postgres/def/types.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 080c0e72..41bb790f 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -417,6 +417,7 @@ pub struct EnumRow { pub values: String, } +#[cfg(feature = "sqlx-dep")] use crate::sqlx_types::postgres::PgRow; impl From<&PgRow> for EnumRow { fn from(row: &PgRow) -> Self { @@ -427,6 +428,7 @@ impl From<&PgRow> for EnumRow { } } +#[cfg(feature = "sqlx-dep")] impl From<&EnumRow> for EnumDef { fn from(row: &EnumRow) -> Self { let fields = row From c63006925bf3ff5d4b5ee191f8d4d973673cb721 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 19 Nov 2021 10:40:01 +0300 Subject: [PATCH 12/31] Switch to using re-exported sqlx types within sea-schema --- src/postgres/def/types.rs | 11 +++-------- src/postgres/discovery/mod.rs | 1 - 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 41bb790f..e21eca8c 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -1,7 +1,6 @@ use sea_query::{backend::PostgresQueryBuilder, extension::postgres::TypeCreateStatement}; -#[cfg(feature = "sqlx-dep")] -use sqlx::Row; +use crate::sqlx_types::{postgres::PgRow, FromRow, Row}; #[cfg(feature = "with-serde")] use serde::{Deserialize, Serialize}; @@ -407,8 +406,7 @@ impl EnumDef { } /// Holds the enum and their values from a `PgRow` -#[cfg(feature = "sqlx-dep")] -#[derive(Debug, sqlx::FromRow)] +#[derive(Debug, FromRow)] pub struct EnumRow { // The name of the enum type pub name: String, @@ -417,18 +415,15 @@ pub struct EnumRow { pub values: String, } -#[cfg(feature = "sqlx-dep")] -use crate::sqlx_types::postgres::PgRow; impl From<&PgRow> for EnumRow { fn from(row: &PgRow) -> Self { - Self { + EnumRow { name: row.get(0), values: row.get(1), } } } -#[cfg(feature = "sqlx-dep")] impl From<&EnumRow> for EnumDef { fn from(row: &EnumRow) -> Self { let fields = row diff --git a/src/postgres/discovery/mod.rs b/src/postgres/discovery/mod.rs index 546ff73f..67724f18 100644 --- a/src/postgres/discovery/mod.rs +++ b/src/postgres/discovery/mod.rs @@ -177,7 +177,6 @@ impl SchemaDiscovery { constraints } - #[cfg(feature = "sqlx-postgres")] pub async fn discover_enums(&self) -> Vec { let mut query = SelectStatement::new(); From 30bb0b4b68af52eb05f88518c35c2a0363607883 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Fri, 19 Nov 2021 16:40:22 +0800 Subject: [PATCH 13/31] Fix CI errors --- src/postgres/def/types.rs | 5 +++-- src/postgres/discovery/mod.rs | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index e21eca8c..38d26921 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -1,6 +1,6 @@ use sea_query::{backend::PostgresQueryBuilder, extension::postgres::TypeCreateStatement}; -use crate::sqlx_types::{postgres::PgRow, FromRow, Row}; +use crate::sqlx_types::{postgres::PgRow, Row}; #[cfg(feature = "with-serde")] use serde::{Deserialize, Serialize}; @@ -406,7 +406,7 @@ impl EnumDef { } /// Holds the enum and their values from a `PgRow` -#[derive(Debug, FromRow)] +#[derive(Debug)] pub struct EnumRow { // The name of the enum type pub name: String, @@ -415,6 +415,7 @@ pub struct EnumRow { pub values: String, } +#[cfg(feature = "sqlx-postgres")] impl From<&PgRow> for EnumRow { fn from(row: &PgRow) -> Self { EnumRow { diff --git a/src/postgres/discovery/mod.rs b/src/postgres/discovery/mod.rs index 67724f18..546ff73f 100644 --- a/src/postgres/discovery/mod.rs +++ b/src/postgres/discovery/mod.rs @@ -177,6 +177,7 @@ impl SchemaDiscovery { constraints } + #[cfg(feature = "sqlx-postgres")] pub async fn discover_enums(&self) -> Vec { let mut query = SelectStatement::new(); From 0e83d2ab89e9cf454617e695655bb18cdd562a73 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 19 Nov 2021 13:34:16 +0300 Subject: [PATCH 14/31] Support enums with crazy names that include special characters --- src/postgres/def/types.rs | 81 ++----------------------- src/postgres/discovery/executor/real.rs | 2 +- src/postgres/discovery/mod.rs | 66 ++++++++++++-------- 3 files changed, 46 insertions(+), 103 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 38d26921..b18f5a22 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -298,58 +298,6 @@ impl Type { _ => None, } } - /// Convert a raw SQL query into a type [Type::Enum] - pub fn enum_from_query(query: &str) -> Type { - use crate::parser::Parser; - use sea_query::{unescape_string, Token}; - - let mut parser = Parser::new(&query); - - let mut type_enum = Type::Enum(EnumDef::default()); - - let mut enum_name = String::default(); - - while let Some(_) = parser.next() { - if parser.last == Some(Token::Unquoted("TYPE".to_owned())) && enum_name.is_empty() { - match parser.curr() { - None => (), - Some(token) => match token { - Token::Quoted(_) => { - if let Some(unquoted) = token.unquote() { - enum_name = unquoted; - } - } - Token::Unquoted(unquoted_token) => enum_name = unquoted_token.to_owned(), - _ => (), - }, - }; - } - - if parser.next_if_punctuation("(") { - while parser.curr().is_some() { - if let Some(word) = parser.next_if_quoted_any() { - type_enum - .get_enum_def_mut() - .values - .push(unescape_string(word.unquote().unwrap().as_str())); - parser.next_if_punctuation(","); - } else if parser.curr_is_unquoted() { - todo!("there can actually be numeric enum values but is very confusing"); - } - if parser.next_if_punctuation(")") { - break; - } - } - } - } - - let attr_len = enum_name.len() as u16; - - type_enum.get_enum_def_mut().typename = enum_name; - type_enum.get_enum_def_mut().attr.length = Some(attr_len); - - type_enum - } } /// Used to ensure enum names and enum fields always implement [sea_query::types::IntoIden] @@ -409,36 +357,17 @@ impl EnumDef { #[derive(Debug)] pub struct EnumRow { // The name of the enum type - pub name: String, - /// The values of the enum type which are concatenated using ` | ` symbol - /// for example `"sans|serif|monospace"` - pub values: String, + pub typname: String, + /// The values of the enum type + pub enumlabel: String, } #[cfg(feature = "sqlx-postgres")] impl From<&PgRow> for EnumRow { fn from(row: &PgRow) -> Self { EnumRow { - name: row.get(0), - values: row.get(1), - } - } -} - -impl From<&EnumRow> for EnumDef { - fn from(row: &EnumRow) -> Self { - let fields = row - .values - .split("|") - .map(|field| field.to_owned()) - .collect::>(); - - Self { - typename: row.name.to_owned(), - attr: StringAttr { - length: Some(row.name.len() as u16), - }, - values: fields, + typname: row.get(0), + enumlabel: row.get(1), } } } diff --git a/src/postgres/discovery/executor/real.rs b/src/postgres/discovery/executor/real.rs index 0df35ead..52ea2146 100644 --- a/src/postgres/discovery/executor/real.rs +++ b/src/postgres/discovery/executor/real.rs @@ -35,7 +35,7 @@ impl Executor { /// Fetches enums from the enum column. There are many ways to do this however, /// this function uses the SQL statement - /// `SELECT type.typname AS name, string_agg(enum.enumlabel, '|') AS value FROM pg_enum AS enum JOIN pg_type AS type ON type.oid = enum.enumtypid GROUP BY type.typname; ` + /// SELECT pg_type.typname, pg_enum.enumlabel FROM pg_type JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid; pub async fn get_enums(&self, select: SelectStatement) -> Vec { let (sql, values) = select.build(PostgresQueryBuilder); debug_print!("{}, {:?}", sql, values); diff --git a/src/postgres/discovery/mod.rs b/src/postgres/discovery/mod.rs index 546ff73f..e0cdba32 100644 --- a/src/postgres/discovery/mod.rs +++ b/src/postgres/discovery/mod.rs @@ -8,6 +8,7 @@ use crate::postgres::query::{ }; use futures::future; use sea_query::{Alias, ColumnRef, Expr, Iden, IntoIden, JoinType, SeaRc, SelectStatement}; +use std::collections::HashMap; mod executor; pub use executor::*; @@ -181,42 +182,55 @@ impl SchemaDiscovery { pub async fn discover_enums(&self) -> Vec { let mut query = SelectStatement::new(); - // SELECT type.typname AS name, string_agg(enum.enumlabel, '|') AS value FROM pg_enum AS enum JOIN pg_type AS type ON type.oid = enum.enumtypid GROUP BY type.typname + // SELECT pg_type.typname, pg_enum.enumlabel FROM pg_type JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid; query - .expr_as( - Expr::col(ColumnRef::TableColumn( - SeaRc::new(Alias::new("type")), - SeaRc::new(Alias::new("typname")), - )), - Alias::new("name"), - ) - .expr_as( - Expr::cust("string_agg(enum.enumlabel, '|')"), - Alias::new("value"), - ) - .from_as(Alias::new("pg_enum"), Alias::new("enum")) - .join_as( - JoinType::Join, - Alias::new("pg_type"), - Alias::new("type"), - Expr::tbl(Alias::new("type"), Alias::new("oid")) - .equals(Alias::new("enum"), Alias::new("enumtypid")), - ) - .group_by_col(ColumnRef::TableColumn( - SeaRc::new(Alias::new("type")), + .expr(Expr::col(ColumnRef::TableColumn( + SeaRc::new(Alias::new("pg_type")), SeaRc::new(Alias::new("typname")), - )); - + ))) + .expr(Expr::col(ColumnRef::TableColumn( + SeaRc::new(Alias::new("pg_enum")), + SeaRc::new(Alias::new("enumlabel")), + ))) + .from(Alias::new("pg_type")) + .join( + JoinType::Join, + Alias::new("pg_enum"), + Expr::tbl(Alias::new("pg_enum"), Alias::new("enumtypid")) + .equals(Alias::new("pg_type"), Alias::new("oid")), + ); let enum_rows = self.executor.get_enums(query).await; let mut enums: Vec = Vec::default(); + let mut temp_enumdef: HashMap = HashMap::new(); + enum_rows.iter().for_each(|enum_row| { - let enum_def: EnumDef = enum_row.into(); + if let Some(entry_exists) = temp_enumdef.get_mut(&enum_row.typname) { + entry_exists.push(enum_row.enumlabel.to_owned()); + } else { + temp_enumdef.insert( + enum_row.typname.to_owned(), + vec![enum_row.enumlabel.to_owned()], + ); + } + }); + + temp_enumdef.into_iter().for_each(|key_value_pair| { + let mut pg_enum = Type::Enum(EnumDef::default()); + pg_enum.get_enum_def_mut().typename = key_value_pair.0.clone(); - enums.push(Type::Enum(enum_def)); + let attr_len = key_value_pair.0.len() as u16; + pg_enum.get_enum_def_mut().attr.length = Some(attr_len); + + pg_enum.get_enum_def_mut().values = key_value_pair.1; + + enums.push(pg_enum); }); enums } } + +type Typname = String; +type EnumValues = Vec; From 134674c11472d4363c5ca58872e9d94d356693f9 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 19 Nov 2021 15:29:08 +0300 Subject: [PATCH 15/31] Add support for ordered Vecs --- src/postgres/def/types.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index b18f5a22..b6403ff7 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -219,7 +219,7 @@ pub struct ArbitraryPrecisionNumericAttr { pub scale: Option, } -#[derive(Clone, Debug, PartialEq, Default)] +#[derive(Clone, Debug, PartialEq, Default, PartialOrd, Eq, Ord)] #[cfg_attr(feature = "with-serde", derive(Serialize, Deserialize))] pub struct StringAttr { pub length: Option, @@ -270,8 +270,17 @@ impl Type { pub fn has_bit_attr(&self) -> bool { matches!(self, Type::Bit(_)) } + /// Get an immutable reference to the [EnumDef] of type [Type::Enum] - pub fn get_enum_def(&self) -> &EnumDef { + pub fn get_enum_def(self) -> EnumDef { + match self { + Type::Enum(def) => def, + _ => panic!("type error"), + } + } + + /// Get an immutable reference to the [EnumDef] of type [Type::Enum] + pub fn get_enum_def_ref(&self) -> &EnumDef { match self { Type::Enum(def) => def, _ => panic!("type error"), @@ -290,14 +299,6 @@ impl Type { pub fn is_enum(&self) -> bool { matches!(self, Type::Enum(_)) } - - /// Returns nome if the type is not an enum `Type::Enum(EnumDef)` - pub fn enum_to_create_statement(&self) -> Option { - match self { - Type::Enum(enum_def) => Some(enum_def.to_sql_query()), - _ => None, - } - } } /// Used to ensure enum names and enum fields always implement [sea_query::types::IntoIden] @@ -311,7 +312,7 @@ impl sea_query::Iden for EnumIden { } /// Defines an enum for the PostgreSQL module -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq, PartialOrd, Ord)] #[cfg_attr(feature = "with-serde", derive(Serialize, Deserialize))] pub struct EnumDef { /// Holds the fields of the `ENUM` @@ -337,7 +338,8 @@ impl EnumDef { } /// Converts the [EnumDef] to a [TypeCreateStatement] - pub fn to_create_statement(&self) -> TypeCreateStatement { + pub fn to_create_statement(&mut self) -> TypeCreateStatement { + self.values.sort(); sea_query::extension::postgres::Type::create() .as_enum(self.typename_impl_iden()) .values(self.values_impl_iden()) @@ -345,7 +347,8 @@ impl EnumDef { } /// Converts the [EnumDef] to a SQL statement - pub fn to_sql_query(&self) -> String { + pub fn to_sql_query(&mut self) -> String { + self.values.sort(); sea_query::extension::postgres::Type::create() .as_enum(self.typename_impl_iden()) .values(self.values_impl_iden()) From d481ed130ba5369e0cd790c38906c33ab03aa036 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 19 Nov 2021 15:30:55 +0300 Subject: [PATCH 16/31] Add tests for discovery of enums Remove redundant module postgres_enum from the tests dir --- Cargo.toml | 1 - tests/live/postgres/Cargo.toml | 3 +- tests/live/postgres/src/main.rs | 60 ++++++++++++++++++++++++++ tests/writer/postgres_enum/Cargo.toml | 12 ------ tests/writer/postgres_enum/Readme.md | 5 --- tests/writer/postgres_enum/src/main.rs | 20 --------- 6 files changed, 62 insertions(+), 39 deletions(-) delete mode 100644 tests/writer/postgres_enum/Cargo.toml delete mode 100644 tests/writer/postgres_enum/Readme.md delete mode 100644 tests/writer/postgres_enum/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 12cf0761..6759a5e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ members = [ "tests/discovery/postgres", "tests/writer/mysql", "tests/writer/postgres", - "tests/writer/postgres_enum", "tests/live/mysql", "tests/live/postgres", ] diff --git a/tests/live/postgres/Cargo.toml b/tests/live/postgres/Cargo.toml index 7fffd599..97ad23c2 100644 --- a/tests/live/postgres/Cargo.toml +++ b/tests/live/postgres/Cargo.toml @@ -6,6 +6,7 @@ publish = false [dependencies] async-std = { version = "1.8", features = [ "attributes" ] } +sea-query = "0.19.0" sea-schema = { path = "../../../", default-features = false, features = [ "sqlx-postgres", "runtime-async-std-native-tls", "discovery", "writer", "debug-print" ] } serde_json = { version = "^1" } -sqlx = { version = "^0" } \ No newline at end of file +sqlx = { version = "^0" } diff --git a/tests/live/postgres/src/main.rs b/tests/live/postgres/src/main.rs index 724f9506..4fd174f9 100644 --- a/tests/live/postgres/src/main.rs +++ b/tests/live/postgres/src/main.rs @@ -53,6 +53,8 @@ async fn main() { println!(); assert_eq!(expected_sql, sql); } + + create_fetch_verify_enum().await; } async fn setup(base_url: &str, db_name: &str) -> Pool { @@ -276,3 +278,61 @@ fn create_cake_table() -> TableCreateStatement { ) .to_owned() } + +/// A test case for PostgreSQL enums in the following steps: +/// +/// Creates an enum with string types that have special characters +/// Add the enum type to the database +/// Discover all enums from the database +/// Assert that the enum created earlier is amongst those discovered +async fn create_fetch_verify_enum() { + use sea_query::{extension::postgres::TypeCreateStatement, Alias, PostgresQueryBuilder}; + + let create_crazy_enum = TypeCreateStatement::new() + .as_enum(Alias::new("crazy_enum")) + .values(vec![ + Alias::new("Astro0%00%8987,.!@#$%^&*()_-+=[]{}\\|.<>/? ``"), + Alias::new("Biology"), + Alias::new("Chemistry"), + Alias::new("Math"), + Alias::new("Physics"), + ]) + .to_string(PostgresQueryBuilder); + + use sqlx::postgres::PgPoolOptions; + + let pool = PgPoolOptions::new() + .max_connections(5) + .connect("postgres://sea:sea@localhost/sakila") + .await + .unwrap(); + + sqlx::query(&create_crazy_enum) + .execute(&pool) + .await + .unwrap(); + + let connection = PgPool::connect("postgres://sea:sea@localhost/sakila") + .await + .unwrap(); + + let enums_discovery = SchemaDiscovery::new(connection, "public") + .discover_enums() + .await; + + dbg!(&enums_discovery); + + let enum_create_statements = enums_discovery + .into_iter() + .map(|enum_type| enum_type.get_enum_def().to_sql_query()) + .collect::>(); + + dbg!(&enum_create_statements); + + assert!( + match enum_create_statements.binary_search(&create_crazy_enum) { + Ok(_) => true, + Err(_) => false, + } + ); +} diff --git a/tests/writer/postgres_enum/Cargo.toml b/tests/writer/postgres_enum/Cargo.toml deleted file mode 100644 index 8e10c841..00000000 --- a/tests/writer/postgres_enum/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "sea-schema-writer-test-postgres-enum" -version = "0.1.0" -edition = "2018" -publish = false - -[dependencies] -async-std = { version = "1.8", features = [ "attributes" ] } -sea-query = "0.18.2" -sea-schema = { path = "../../../", default-features = false, features = [ "sqlx-postgres", "runtime-async-std-native-tls", "discovery", "writer", "debug-print" ] } -serde_json = { version = "^1" } -sqlx = { version = "^0" } diff --git a/tests/writer/postgres_enum/Readme.md b/tests/writer/postgres_enum/Readme.md deleted file mode 100644 index 6946a346..00000000 --- a/tests/writer/postgres_enum/Readme.md +++ /dev/null @@ -1,5 +0,0 @@ -# Run - -```sh -cargo run > schema.sql -``` \ No newline at end of file diff --git a/tests/writer/postgres_enum/src/main.rs b/tests/writer/postgres_enum/src/main.rs deleted file mode 100644 index eee689e6..00000000 --- a/tests/writer/postgres_enum/src/main.rs +++ /dev/null @@ -1,20 +0,0 @@ -use sea_schema::postgres::{def::Type, discovery::SchemaDiscovery}; -use sqlx::PgPool; - -#[async_std::main] -async fn main() { - let connection = PgPool::connect("postgres://sea:sea@localhost/sakila") - .await - .unwrap(); - - let enums_discovery = SchemaDiscovery::new(connection, "public") - .discover_enums() - .await; - - // Assert that all enums in the database are equal to their - // raw SQL queries - enums_discovery.iter().for_each(|enum_type| { - let to_sql_query = enum_type.get_enum_def().to_sql_query(); - assert_eq!(enum_type, &Type::enum_from_query(&to_sql_query)); - }); -} From dbb03b031e4f8b31f117c281d5075daa2aa03879 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Fri, 19 Nov 2021 15:46:30 +0300 Subject: [PATCH 17/31] Ensure a custom enum type is dropped first from the database so that CI runs successfully --- tests/live/postgres/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/live/postgres/src/main.rs b/tests/live/postgres/src/main.rs index 4fd174f9..1a98b3bd 100644 --- a/tests/live/postgres/src/main.rs +++ b/tests/live/postgres/src/main.rs @@ -307,6 +307,11 @@ async fn create_fetch_verify_enum() { .await .unwrap(); + sqlx::query("DROP TYPE IF EXISTS crazy_enum;") + .execute(&pool) + .await + .unwrap(); + sqlx::query(&create_crazy_enum) .execute(&pool) .await From 2e68190ced6adb1968fd7d00b54fecbb15db0b35 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Mon, 22 Nov 2021 12:42:26 +0800 Subject: [PATCH 18/31] Refactoring --- src/postgres/def/types.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index b6403ff7..7431c870 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -348,11 +348,7 @@ impl EnumDef { /// Converts the [EnumDef] to a SQL statement pub fn to_sql_query(&mut self) -> String { - self.values.sort(); - sea_query::extension::postgres::Type::create() - .as_enum(self.typename_impl_iden()) - .values(self.values_impl_iden()) - .to_string(PostgresQueryBuilder) + self.to_create_statement().to_string(PostgresQueryBuilder) } } From 48b9ff3397746cfabd523606f586de72c8c1ac10 Mon Sep 17 00:00:00 2001 From: Charles Chege Date: Mon, 22 Nov 2021 08:46:47 +0300 Subject: [PATCH 19/31] Ensures the writer derives the enum name from the EnumDef --- src/postgres/writer/column.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/postgres/writer/column.rs b/src/postgres/writer/column.rs index 0231c3e7..9ca01552 100644 --- a/src/postgres/writer/column.rs +++ b/src/postgres/writer/column.rs @@ -228,7 +228,7 @@ impl ColumnInfo { col_def.custom(Alias::new(s)); } Type::Enum(def) => { - col_def.custom(Alias::new("enum")); + col_def.custom(Alias::new(&def.typename)); } }; col_def From 689fe9bd0286b1be5fcfce9a69e3e9727c78eb9f Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Mon, 22 Nov 2021 23:05:22 +0800 Subject: [PATCH 20/31] Refactoring --- src/postgres/def/types.rs | 31 +-------- src/postgres/discovery/mod.rs | 88 ++++++++++---------------- src/postgres/query/enumeration.rs | 61 ++++++++++++++++++ src/postgres/query/mod.rs | 2 + tests/live/postgres/src/main.rs | 102 ++++++++++-------------------- 5 files changed, 133 insertions(+), 151 deletions(-) create mode 100644 src/postgres/query/enumeration.rs diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 7431c870..acc53911 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -271,32 +271,7 @@ impl Type { matches!(self, Type::Bit(_)) } - /// Get an immutable reference to the [EnumDef] of type [Type::Enum] - pub fn get_enum_def(self) -> EnumDef { - match self { - Type::Enum(def) => def, - _ => panic!("type error"), - } - } - - /// Get an immutable reference to the [EnumDef] of type [Type::Enum] - pub fn get_enum_def_ref(&self) -> &EnumDef { - match self { - Type::Enum(def) => def, - _ => panic!("type error"), - } - } - - /// Get a mutable reference to the [EnumDef] of type [Type::Enum] - pub fn get_enum_def_mut(&mut self) -> &mut EnumDef { - match self { - Type::Enum(def) => def, - _ => panic!("type error"), - } - } - - /// Is the type given an enum - pub fn is_enum(&self) -> bool { + pub fn has_enum_attr(&self) -> bool { matches!(self, Type::Enum(_)) } } @@ -312,13 +287,11 @@ impl sea_query::Iden for EnumIden { } /// Defines an enum for the PostgreSQL module -#[derive(Clone, Debug, Default, Eq, PartialEq, PartialOrd, Ord)] +#[derive(Clone, Debug, PartialEq, Default)] #[cfg_attr(feature = "with-serde", derive(Serialize, Deserialize))] pub struct EnumDef { /// Holds the fields of the `ENUM` pub values: Vec, - /// Defines the length of the name describing the [Type::Enum] - pub attr: StringAttr, /// Defines the name of the PostgreSQL enum identifier pub typename: String, } diff --git a/src/postgres/discovery/mod.rs b/src/postgres/discovery/mod.rs index e0cdba32..ace2bcb3 100644 --- a/src/postgres/discovery/mod.rs +++ b/src/postgres/discovery/mod.rs @@ -4,7 +4,8 @@ use crate::debug_print; use crate::postgres::def::*; use crate::postgres::parser::parse_table_constraint_query_results; use crate::postgres::query::{ - ColumnQueryResult, SchemaQueryBuilder, TableConstraintsQueryResult, TableQueryResult, + ColumnQueryResult, EnumQueryResult, SchemaQueryBuilder, TableConstraintsQueryResult, + TableQueryResult, }; use futures::future; use sea_query::{Alias, ColumnRef, Expr, Iden, IntoIden, JoinType, SeaRc, SelectStatement}; @@ -178,59 +179,36 @@ impl SchemaDiscovery { constraints } - #[cfg(feature = "sqlx-postgres")] - pub async fn discover_enums(&self) -> Vec { - let mut query = SelectStatement::new(); - - // SELECT pg_type.typname, pg_enum.enumlabel FROM pg_type JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid; - query - .expr(Expr::col(ColumnRef::TableColumn( - SeaRc::new(Alias::new("pg_type")), - SeaRc::new(Alias::new("typname")), - ))) - .expr(Expr::col(ColumnRef::TableColumn( - SeaRc::new(Alias::new("pg_enum")), - SeaRc::new(Alias::new("enumlabel")), - ))) - .from(Alias::new("pg_type")) - .join( - JoinType::Join, - Alias::new("pg_enum"), - Expr::tbl(Alias::new("pg_enum"), Alias::new("enumtypid")) - .equals(Alias::new("pg_type"), Alias::new("oid")), - ); - let enum_rows = self.executor.get_enums(query).await; - - let mut enums: Vec = Vec::default(); - - let mut temp_enumdef: HashMap = HashMap::new(); - - enum_rows.iter().for_each(|enum_row| { - if let Some(entry_exists) = temp_enumdef.get_mut(&enum_row.typname) { - entry_exists.push(enum_row.enumlabel.to_owned()); - } else { - temp_enumdef.insert( - enum_row.typname.to_owned(), - vec![enum_row.enumlabel.to_owned()], - ); - } - }); - - temp_enumdef.into_iter().for_each(|key_value_pair| { - let mut pg_enum = Type::Enum(EnumDef::default()); - pg_enum.get_enum_def_mut().typename = key_value_pair.0.clone(); - - let attr_len = key_value_pair.0.len() as u16; - pg_enum.get_enum_def_mut().attr.length = Some(attr_len); - - pg_enum.get_enum_def_mut().values = key_value_pair.1; - - enums.push(pg_enum); - }); - - enums + pub async fn discover_enums(&self) -> Vec { + let rows = self.executor.fetch_all(self.query.query_enums()).await; + + let enum_rows: Vec = rows + .iter() + .map(|row| { + let result: EnumQueryResult = row.into(); + debug_print!("{:?}", result); + return result; + }) + .collect(); + + let map = enum_rows.into_iter().fold( + HashMap::new(), + |mut map: HashMap>, + EnumQueryResult { + typename, + enumlabel, + }| { + if let Some(entry_exists) = map.get_mut(&typename) { + entry_exists.push(enumlabel); + } else { + map.insert(typename, vec![enumlabel]); + } + map + }, + ); + + map.into_iter() + .map(|(typename, values)| EnumDef { values, typename }) + .collect() } } - -type Typname = String; -type EnumValues = Vec; diff --git a/src/postgres/query/enumeration.rs b/src/postgres/query/enumeration.rs new file mode 100644 index 00000000..a1b5dac2 --- /dev/null +++ b/src/postgres/query/enumeration.rs @@ -0,0 +1,61 @@ +use super::{InformationSchema, SchemaQueryBuilder}; +use crate::sqlx_types::postgres::PgRow; +use sea_query::{Expr, Iden, Query, SeaRc, SelectStatement}; + +#[derive(Debug, sea_query::Iden)] +pub enum PgType { + #[iden = "pg_type"] + Table, + #[iden = "typname"] + TypeName, + #[iden = "oid"] + Oid, +} + +#[derive(Debug, sea_query::Iden)] +pub enum PgEnum { + #[iden = "pg_enum"] + Table, + #[iden = "enumlabel"] + EnumLabel, + #[iden = "enumtypid"] + EnumTypeId, +} + +#[derive(Debug, Default)] +pub struct EnumQueryResult { + pub typename: String, + pub enumlabel: String, +} + +impl SchemaQueryBuilder { + pub fn query_enums(&self) -> SelectStatement { + Query::select() + .column((PgType::Table, PgType::TypeName)) + .column((PgEnum::Table, PgEnum::EnumLabel)) + .from(PgType::Table) + .inner_join( + PgEnum::Table, + Expr::tbl(PgEnum::Table, PgEnum::EnumTypeId).equals(PgType::Table, PgType::Oid), + ) + .take() + } +} + +#[cfg(feature = "sqlx-postgres")] +impl From<&PgRow> for EnumQueryResult { + fn from(row: &PgRow) -> Self { + use crate::sqlx_types::Row; + Self { + typename: row.get(0), + enumlabel: row.get(1), + } + } +} + +#[cfg(not(feature = "sqlx-postgres"))] +impl From<&PgRow> for EnumQueryResult { + fn from(row: &PgRow) -> Self { + Self::default() + } +} diff --git a/src/postgres/query/mod.rs b/src/postgres/query/mod.rs index 78935ef0..f66212ae 100644 --- a/src/postgres/query/mod.rs +++ b/src/postgres/query/mod.rs @@ -1,11 +1,13 @@ pub mod char_set; pub mod column; pub mod constraints; +pub mod enumeration; pub mod schema; pub mod table; pub use char_set::*; pub use column::*; pub use constraints::*; +pub use enumeration::*; pub use schema::*; pub use table::*; diff --git a/tests/live/postgres/src/main.rs b/tests/live/postgres/src/main.rs index 1a98b3bd..541d6c9c 100644 --- a/tests/live/postgres/src/main.rs +++ b/tests/live/postgres/src/main.rs @@ -2,9 +2,10 @@ use std::collections::HashMap; use sea_schema::postgres::{def::TableDef, discovery::SchemaDiscovery}; use sea_schema::sea_query::{ - Alias, ColumnDef, ForeignKey, ForeignKeyAction, Index, PostgresQueryBuilder, Table, - TableCreateStatement, + extension::postgres::Type, Alias, ColumnDef, ForeignKey, ForeignKeyAction, Index, + PostgresQueryBuilder, Table, TableCreateStatement, }; +use sqlx::pool::PoolConnection; use sqlx::{PgPool, Pool, Postgres}; #[cfg_attr(test, async_std::test)] @@ -30,7 +31,23 @@ async fn main() { sqlx::query(&sql).execute(&mut executor).await.unwrap(); } - let schema_discovery = SchemaDiscovery::new(connection, "public"); + let create_enum_stmt = Type::create() + .as_enum(Alias::new("crazy_enum")) + .values(vec![ + Alias::new("Astro0%00%8987,.!@#$%^&*()_-+=[]{}\\|.<>/? ``"), + Alias::new("Biology"), + Alias::new("Chemistry"), + Alias::new("Math"), + Alias::new("Physics"), + ]) + .to_string(PostgresQueryBuilder); + + sqlx::query(&create_enum_stmt) + .execute(&mut executor) + .await + .unwrap(); + + let schema_discovery = SchemaDiscovery::new(connection.clone(), "public"); let schema = schema_discovery.discover().await; @@ -54,7 +71,21 @@ async fn main() { assert_eq!(expected_sql, sql); } - create_fetch_verify_enum().await; + let schema_discovery = SchemaDiscovery::new(connection, "public"); + + let enum_defs = schema_discovery.discover_enums().await; + + dbg!(&enum_defs); + + let enum_create_statements = enum_defs + .into_iter() + .map(|mut enum_type| enum_type.to_sql_query()) + .collect::>(); + + dbg!(&create_enum_stmt); + dbg!(&enum_create_statements); + + assert_eq!(create_enum_stmt, enum_create_statements[0]); } async fn setup(base_url: &str, db_name: &str) -> Pool { @@ -278,66 +309,3 @@ fn create_cake_table() -> TableCreateStatement { ) .to_owned() } - -/// A test case for PostgreSQL enums in the following steps: -/// -/// Creates an enum with string types that have special characters -/// Add the enum type to the database -/// Discover all enums from the database -/// Assert that the enum created earlier is amongst those discovered -async fn create_fetch_verify_enum() { - use sea_query::{extension::postgres::TypeCreateStatement, Alias, PostgresQueryBuilder}; - - let create_crazy_enum = TypeCreateStatement::new() - .as_enum(Alias::new("crazy_enum")) - .values(vec![ - Alias::new("Astro0%00%8987,.!@#$%^&*()_-+=[]{}\\|.<>/? ``"), - Alias::new("Biology"), - Alias::new("Chemistry"), - Alias::new("Math"), - Alias::new("Physics"), - ]) - .to_string(PostgresQueryBuilder); - - use sqlx::postgres::PgPoolOptions; - - let pool = PgPoolOptions::new() - .max_connections(5) - .connect("postgres://sea:sea@localhost/sakila") - .await - .unwrap(); - - sqlx::query("DROP TYPE IF EXISTS crazy_enum;") - .execute(&pool) - .await - .unwrap(); - - sqlx::query(&create_crazy_enum) - .execute(&pool) - .await - .unwrap(); - - let connection = PgPool::connect("postgres://sea:sea@localhost/sakila") - .await - .unwrap(); - - let enums_discovery = SchemaDiscovery::new(connection, "public") - .discover_enums() - .await; - - dbg!(&enums_discovery); - - let enum_create_statements = enums_discovery - .into_iter() - .map(|enum_type| enum_type.get_enum_def().to_sql_query()) - .collect::>(); - - dbg!(&enum_create_statements); - - assert!( - match enum_create_statements.binary_search(&create_crazy_enum) { - Ok(_) => true, - Err(_) => false, - } - ); -} From 172f2a3ec97708c7979fa5ac3a4e087aa3d0a440 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Mon, 22 Nov 2021 23:09:39 +0800 Subject: [PATCH 21/31] Refactoring --- src/postgres/def/types.rs | 10 ---------- src/postgres/discovery/executor/real.rs | 23 ----------------------- 2 files changed, 33 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index acc53911..a68819d3 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -333,13 +333,3 @@ pub struct EnumRow { /// The values of the enum type pub enumlabel: String, } - -#[cfg(feature = "sqlx-postgres")] -impl From<&PgRow> for EnumRow { - fn from(row: &PgRow) -> Self { - EnumRow { - typname: row.get(0), - enumlabel: row.get(1), - } - } -} diff --git a/src/postgres/discovery/executor/real.rs b/src/postgres/discovery/executor/real.rs index 52ea2146..36b71ccd 100644 --- a/src/postgres/discovery/executor/real.rs +++ b/src/postgres/discovery/executor/real.rs @@ -32,27 +32,4 @@ impl Executor { .await .unwrap() } - - /// Fetches enums from the enum column. There are many ways to do this however, - /// this function uses the SQL statement - /// SELECT pg_type.typname, pg_enum.enumlabel FROM pg_type JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid; - pub async fn get_enums(&self, select: SelectStatement) -> Vec { - let (sql, values) = select.build(PostgresQueryBuilder); - debug_print!("{}, {:?}", sql, values); - - let query = bind_query(sqlx::query(&sql), &values); - - let rows = query - .fetch_all(&mut self.pool.acquire().await.unwrap()) - .await - .unwrap(); - - rows.iter() - .map(|pg_row| { - let column: EnumRow = pg_row.into(); - - column - }) - .collect::>() - } } From 3704da45c046efd9ad6a7f6486719259071fd4e1 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Mon, 22 Nov 2021 23:15:01 +0800 Subject: [PATCH 22/31] Refactoring --- src/postgres/writer/column.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/postgres/writer/column.rs b/src/postgres/writer/column.rs index 9ca01552..f2b29636 100644 --- a/src/postgres/writer/column.rs +++ b/src/postgres/writer/column.rs @@ -228,7 +228,7 @@ impl ColumnInfo { col_def.custom(Alias::new(s)); } Type::Enum(def) => { - col_def.custom(Alias::new(&def.typename)); + col_def.custom(def.typename_impl_iden()); } }; col_def From 37a452c1d6dfc5ead80986e0e6c93c542fb4ef0b Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Mon, 22 Nov 2021 23:15:22 +0800 Subject: [PATCH 23/31] Testing enum column... --- tests/live/postgres/src/main.rs | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/live/postgres/src/main.rs b/tests/live/postgres/src/main.rs index 541d6c9c..8fb01c02 100644 --- a/tests/live/postgres/src/main.rs +++ b/tests/live/postgres/src/main.rs @@ -14,6 +14,22 @@ async fn main() { let connection = setup("postgres://sea:sea@localhost", "sea-schema").await; let mut executor = connection.acquire().await.unwrap(); + let create_enum_stmt = Type::create() + .as_enum(Alias::new("crazy_enum")) + .values(vec![ + Alias::new("Astro0%00%8987,.!@#$%^&*()_-+=[]{}\\|.<>/? ``"), + Alias::new("Biology"), + Alias::new("Chemistry"), + Alias::new("Math"), + Alias::new("Physics"), + ]) + .to_string(PostgresQueryBuilder); + + sqlx::query(&create_enum_stmt) + .execute(&mut executor) + .await + .unwrap(); + let tbl_create_stmts = vec![ create_bakery_table(), create_baker_table(), @@ -31,22 +47,6 @@ async fn main() { sqlx::query(&sql).execute(&mut executor).await.unwrap(); } - let create_enum_stmt = Type::create() - .as_enum(Alias::new("crazy_enum")) - .values(vec![ - Alias::new("Astro0%00%8987,.!@#$%^&*()_-+=[]{}\\|.<>/? ``"), - Alias::new("Biology"), - Alias::new("Chemistry"), - Alias::new("Math"), - Alias::new("Physics"), - ]) - .to_string(PostgresQueryBuilder); - - sqlx::query(&create_enum_stmt) - .execute(&mut executor) - .await - .unwrap(); - let schema_discovery = SchemaDiscovery::new(connection.clone(), "public"); let schema = schema_discovery.discover().await; @@ -123,6 +123,7 @@ fn create_bakery_table() -> TableCreateStatement { ) .col(ColumnDef::new(Alias::new("name")).string()) .col(ColumnDef::new(Alias::new("profit_margin")).double()) + .col(ColumnDef::new(Alias::new("crazy_enum_col")).custom(Alias::new("crazy_enum"))) .primary_key( Index::create() .primary() From 2aac450d4d36fec91b81b686c126689f2ffb78e7 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Mon, 22 Nov 2021 23:19:38 +0800 Subject: [PATCH 24/31] Refactoring --- src/postgres/def/types.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index a68819d3..02834c59 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -1,7 +1,5 @@ use sea_query::{backend::PostgresQueryBuilder, extension::postgres::TypeCreateStatement}; -use crate::sqlx_types::{postgres::PgRow, Row}; - #[cfg(feature = "with-serde")] use serde::{Deserialize, Serialize}; From 35879c6a1f1e470ab7bcfe54a71723c934bcb662 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Mon, 22 Nov 2021 23:21:05 +0800 Subject: [PATCH 25/31] Refactoring --- tests/live/postgres/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/live/postgres/Cargo.toml b/tests/live/postgres/Cargo.toml index 97ad23c2..dc685dc2 100644 --- a/tests/live/postgres/Cargo.toml +++ b/tests/live/postgres/Cargo.toml @@ -6,7 +6,6 @@ publish = false [dependencies] async-std = { version = "1.8", features = [ "attributes" ] } -sea-query = "0.19.0" sea-schema = { path = "../../../", default-features = false, features = [ "sqlx-postgres", "runtime-async-std-native-tls", "discovery", "writer", "debug-print" ] } serde_json = { version = "^1" } sqlx = { version = "^0" } From d2ce20876767b55b0b9d9b6aed5d9dfc385fea75 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Mon, 22 Nov 2021 23:37:51 +0800 Subject: [PATCH 26/31] Refactoring --- src/postgres/def/types.rs | 11 +---------- src/postgres/discovery/executor/real.rs | 1 - tests/live/postgres/src/main.rs | 1 - 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 02834c59..099e7a0f 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -217,7 +217,7 @@ pub struct ArbitraryPrecisionNumericAttr { pub scale: Option, } -#[derive(Clone, Debug, PartialEq, Default, PartialOrd, Eq, Ord)] +#[derive(Clone, Debug, PartialEq, Default)] #[cfg_attr(feature = "with-serde", derive(Serialize, Deserialize))] pub struct StringAttr { pub length: Option, @@ -322,12 +322,3 @@ impl EnumDef { self.to_create_statement().to_string(PostgresQueryBuilder) } } - -/// Holds the enum and their values from a `PgRow` -#[derive(Debug)] -pub struct EnumRow { - // The name of the enum type - pub typname: String, - /// The values of the enum type - pub enumlabel: String, -} diff --git a/src/postgres/discovery/executor/real.rs b/src/postgres/discovery/executor/real.rs index 36b71ccd..84cc35d4 100644 --- a/src/postgres/discovery/executor/real.rs +++ b/src/postgres/discovery/executor/real.rs @@ -1,4 +1,3 @@ -use crate::postgres::def::EnumRow; use sea_query::{PostgresQueryBuilder, SelectStatement}; use sqlx::{postgres::PgRow, PgPool}; diff --git a/tests/live/postgres/src/main.rs b/tests/live/postgres/src/main.rs index 8fb01c02..ce9a4604 100644 --- a/tests/live/postgres/src/main.rs +++ b/tests/live/postgres/src/main.rs @@ -5,7 +5,6 @@ use sea_schema::sea_query::{ extension::postgres::Type, Alias, ColumnDef, ForeignKey, ForeignKeyAction, Index, PostgresQueryBuilder, Table, TableCreateStatement, }; -use sqlx::pool::PoolConnection; use sqlx::{PgPool, Pool, Postgres}; #[cfg_attr(test, async_std::test)] From c6e99121340d57e08546373606cc1f1257703f5c Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Tue, 23 Nov 2021 10:51:11 +0800 Subject: [PATCH 27/31] Fix test --- src/postgres/def/types.rs | 3 +-- src/postgres/parser/column.rs | 17 +++++++++++++++++ src/postgres/query/column.rs | 4 ++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 099e7a0f..5df4e3c2 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -170,7 +170,6 @@ impl Type { "time with time zone" => Type::TimeWithTimeZone(TimeAttr::default()), "interval" => Type::Interval(IntervalAttr::default()), "boolean" => Type::Boolean, - // "" => Type::Enum, "point" => Type::Point, "line" => Type::Line, "lseg" => Type::Lseg, @@ -198,7 +197,7 @@ impl Type { "daterange" => Type::DateRange, // "" => Type::Domain, "pg_lsn" => Type::PgLsn, - "enum" => Type::Enum(EnumDef::default()), + "user-defined" => Type::Enum(EnumDef::default()), _ => Type::Unknown(name.to_owned()), } diff --git a/src/postgres/parser/column.rs b/src/postgres/parser/column.rs index 33e9ec4b..61e8ac1f 100644 --- a/src/postgres/parser/column.rs +++ b/src/postgres/parser/column.rs @@ -41,6 +41,9 @@ pub fn parse_column_type(result: &ColumnQueryResult) -> ColumnType { if ctype.has_bit_attr() { ctype = parse_bit_attributes(result.character_maximum_length, ctype); } + if ctype.has_enum_attr() { + ctype = parse_enum_attributes(result.udt_name.as_ref(), ctype); + } ctype } @@ -165,3 +168,17 @@ pub fn parse_bit_attributes( ctype } + +pub fn parse_enum_attributes(udt_name: Option<&String>, mut ctype: ColumnType) -> ColumnType { + match ctype { + Type::Enum(ref mut def) => { + def.typename = match udt_name { + None => panic!("parse_enum_attributes(_) received an empty udt_name"), + Some(typename) => typename.to_string(), + }; + } + _ => panic!("parse_enum_attributes(_) received a type that does not have EnumDef"), + }; + + ctype +} diff --git a/src/postgres/query/column.rs b/src/postgres/query/column.rs index e1e701a7..998fc0d6 100644 --- a/src/postgres/query/column.rs +++ b/src/postgres/query/column.rs @@ -64,6 +64,8 @@ pub struct ColumnQueryResult { pub interval_type: Option, pub interval_precision: Option, + + pub udt_name: Option, } impl SchemaQueryBuilder { @@ -88,6 +90,7 @@ impl SchemaQueryBuilder { ColumnsField::DatetimePrecision, ColumnsField::IntervalType, ColumnsField::IntervalPrecision, + ColumnsField::UdtName, ]) .from((InformationSchema::Schema, InformationSchema::Columns)) .and_where(Expr::col(ColumnsField::TableSchema).eq(schema.to_string())) @@ -115,6 +118,7 @@ impl From<&PgRow> for ColumnQueryResult { datetime_precision: row.get(11), interval_type: row.get(12), interval_precision: row.get(13), + udt_name: row.get(14), } } } From 3b3a49dc17e702f784c64b33ac0603630107bbf9 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Tue, 23 Nov 2021 11:00:18 +0800 Subject: [PATCH 28/31] Refactoring --- src/postgres/discovery/mod.rs | 6 +++--- tests/live/postgres/src/main.rs | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/postgres/discovery/mod.rs b/src/postgres/discovery/mod.rs index ace2bcb3..22ed1dd6 100644 --- a/src/postgres/discovery/mod.rs +++ b/src/postgres/discovery/mod.rs @@ -32,12 +32,12 @@ impl SchemaDiscovery { } } - pub async fn discover(mut self) -> Schema { + pub async fn discover(&self) -> Schema { let tables = self.discover_tables().await; let tables = future::join_all( tables .into_iter() - .map(|t| (&self, t)) + .map(|t| (self, t)) .map(Self::discover_table_static), ) .await; @@ -48,7 +48,7 @@ impl SchemaDiscovery { } } - pub async fn discover_tables(&mut self) -> Vec { + pub async fn discover_tables(&self) -> Vec { let rows = self .executor .fetch_all(self.query.query_tables(self.schema.clone())) diff --git a/tests/live/postgres/src/main.rs b/tests/live/postgres/src/main.rs index ce9a4604..fc9ca35b 100644 --- a/tests/live/postgres/src/main.rs +++ b/tests/live/postgres/src/main.rs @@ -46,7 +46,7 @@ async fn main() { sqlx::query(&sql).execute(&mut executor).await.unwrap(); } - let schema_discovery = SchemaDiscovery::new(connection.clone(), "public"); + let schema_discovery = SchemaDiscovery::new(connection, "public"); let schema = schema_discovery.discover().await; @@ -70,8 +70,6 @@ async fn main() { assert_eq!(expected_sql, sql); } - let schema_discovery = SchemaDiscovery::new(connection, "public"); - let enum_defs = schema_discovery.discover_enums().await; dbg!(&enum_defs); From 4833b9ebae5248471f5e316f48d97349cde2fb54 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Tue, 23 Nov 2021 11:01:15 +0800 Subject: [PATCH 29/31] Refactoring --- tests/live/postgres/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/live/postgres/Cargo.toml b/tests/live/postgres/Cargo.toml index dc685dc2..7fffd599 100644 --- a/tests/live/postgres/Cargo.toml +++ b/tests/live/postgres/Cargo.toml @@ -8,4 +8,4 @@ publish = false async-std = { version = "1.8", features = [ "attributes" ] } sea-schema = { path = "../../../", default-features = false, features = [ "sqlx-postgres", "runtime-async-std-native-tls", "discovery", "writer", "debug-print" ] } serde_json = { version = "^1" } -sqlx = { version = "^0" } +sqlx = { version = "^0" } \ No newline at end of file From 30ae58b4a90c4e198d44496d3e375f057c7bbc4e Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Tue, 23 Nov 2021 11:18:39 +0800 Subject: [PATCH 30/31] Refactoring --- src/postgres/def/types.rs | 59 +++++------------------------- src/postgres/query/enumeration.rs | 4 +- src/postgres/writer/column.rs | 2 +- src/postgres/writer/enumeration.rs | 15 ++++++++ src/postgres/writer/mod.rs | 2 + tests/live/postgres/src/main.rs | 6 +-- 6 files changed, 34 insertions(+), 54 deletions(-) create mode 100644 src/postgres/writer/enumeration.rs diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 5df4e3c2..7a6fd705 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -241,6 +241,16 @@ pub struct BitAttr { pub length: Option, } +/// Defines an enum for the PostgreSQL module +#[derive(Clone, Debug, PartialEq, Default)] +#[cfg_attr(feature = "with-serde", derive(Serialize, Deserialize))] +pub struct EnumDef { + /// Holds the fields of the `ENUM` + pub values: Vec, + /// Defines the name of the PostgreSQL enum identifier + pub typename: String, +} + impl Type { pub fn has_numeric_attr(&self) -> bool { matches!(self, Type::Numeric(_) | Type::Decimal(_)) @@ -272,52 +282,3 @@ impl Type { matches!(self, Type::Enum(_)) } } - -/// Used to ensure enum names and enum fields always implement [sea_query::types::IntoIden] -#[derive(Debug)] -pub struct EnumIden(pub String); - -impl sea_query::Iden for EnumIden { - fn unquoted(&self, s: &mut dyn std::fmt::Write) { - write!(s, "{}", self.0).unwrap(); - } -} - -/// Defines an enum for the PostgreSQL module -#[derive(Clone, Debug, PartialEq, Default)] -#[cfg_attr(feature = "with-serde", derive(Serialize, Deserialize))] -pub struct EnumDef { - /// Holds the fields of the `ENUM` - pub values: Vec, - /// Defines the name of the PostgreSQL enum identifier - pub typename: String, -} - -impl EnumDef { - /// Implements [sea_query::types::IntoIden] for the Enum name - pub fn typename_impl_iden(&self) -> EnumIden { - EnumIden(self.typename.to_owned()) - } - - /// Implements [sea_query::types::IntoIden] for the Enum fields - pub fn values_impl_iden(&self) -> Vec { - self.values - .iter() - .map(|iden| EnumIden(iden.to_owned())) - .collect::>() - } - - /// Converts the [EnumDef] to a [TypeCreateStatement] - pub fn to_create_statement(&mut self) -> TypeCreateStatement { - self.values.sort(); - sea_query::extension::postgres::Type::create() - .as_enum(self.typename_impl_iden()) - .values(self.values_impl_iden()) - .clone() - } - - /// Converts the [EnumDef] to a SQL statement - pub fn to_sql_query(&mut self) -> String { - self.to_create_statement().to_string(PostgresQueryBuilder) - } -} diff --git a/src/postgres/query/enumeration.rs b/src/postgres/query/enumeration.rs index a1b5dac2..ac22ed8f 100644 --- a/src/postgres/query/enumeration.rs +++ b/src/postgres/query/enumeration.rs @@ -1,6 +1,6 @@ use super::{InformationSchema, SchemaQueryBuilder}; use crate::sqlx_types::postgres::PgRow; -use sea_query::{Expr, Iden, Query, SeaRc, SelectStatement}; +use sea_query::{Expr, Iden, Order, Query, SeaRc, SelectStatement}; #[derive(Debug, sea_query::Iden)] pub enum PgType { @@ -38,6 +38,8 @@ impl SchemaQueryBuilder { PgEnum::Table, Expr::tbl(PgEnum::Table, PgEnum::EnumTypeId).equals(PgType::Table, PgType::Oid), ) + .order_by((PgType::Table, PgType::TypeName), Order::Asc) + .order_by((PgEnum::Table, PgEnum::EnumLabel), Order::Asc) .take() } } diff --git a/src/postgres/writer/column.rs b/src/postgres/writer/column.rs index f2b29636..df53c219 100644 --- a/src/postgres/writer/column.rs +++ b/src/postgres/writer/column.rs @@ -228,7 +228,7 @@ impl ColumnInfo { col_def.custom(Alias::new(s)); } Type::Enum(def) => { - col_def.custom(def.typename_impl_iden()); + col_def.custom(Alias::new(def.typename.as_str())); } }; col_def diff --git a/src/postgres/writer/enumeration.rs b/src/postgres/writer/enumeration.rs new file mode 100644 index 00000000..939cf895 --- /dev/null +++ b/src/postgres/writer/enumeration.rs @@ -0,0 +1,15 @@ +use crate::postgres::def::EnumDef; +use sea_query::{ + extension::postgres::{Type, TypeCreateStatement}, + Alias, +}; + +impl EnumDef { + /// Converts the [EnumDef] to a [TypeCreateStatement] + pub fn write(&self) -> TypeCreateStatement { + Type::create() + .as_enum(Alias::new(self.typename.as_str())) + .values(self.values.iter().map(|val| Alias::new(val.as_str()))) + .to_owned() + } +} diff --git a/src/postgres/writer/mod.rs b/src/postgres/writer/mod.rs index 818d9dfc..732480f4 100644 --- a/src/postgres/writer/mod.rs +++ b/src/postgres/writer/mod.rs @@ -1,11 +1,13 @@ mod column; mod constraints; +mod enumeration; mod schema; mod table; mod types; pub use column::*; pub use constraints::*; +pub use enumeration::*; pub use schema::*; pub use table::*; pub use types::*; diff --git a/tests/live/postgres/src/main.rs b/tests/live/postgres/src/main.rs index fc9ca35b..cfe2a4a3 100644 --- a/tests/live/postgres/src/main.rs +++ b/tests/live/postgres/src/main.rs @@ -74,10 +74,10 @@ async fn main() { dbg!(&enum_defs); - let enum_create_statements = enum_defs + let enum_create_statements: Vec = enum_defs .into_iter() - .map(|mut enum_type| enum_type.to_sql_query()) - .collect::>(); + .map(|enum_def| enum_def.write().to_string(PostgresQueryBuilder)) + .collect(); dbg!(&create_enum_stmt); dbg!(&enum_create_statements); From 9c3ffc2519408470aa6d40d924c68dde59833357 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Tue, 23 Nov 2021 11:23:29 +0800 Subject: [PATCH 31/31] Cleanup --- src/postgres/def/types.rs | 2 -- src/postgres/discovery/mod.rs | 2 +- src/postgres/query/enumeration.rs | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/postgres/def/types.rs b/src/postgres/def/types.rs index 7a6fd705..e2057f62 100644 --- a/src/postgres/def/types.rs +++ b/src/postgres/def/types.rs @@ -1,5 +1,3 @@ -use sea_query::{backend::PostgresQueryBuilder, extension::postgres::TypeCreateStatement}; - #[cfg(feature = "with-serde")] use serde::{Deserialize, Serialize}; diff --git a/src/postgres/discovery/mod.rs b/src/postgres/discovery/mod.rs index 22ed1dd6..f8c5b622 100644 --- a/src/postgres/discovery/mod.rs +++ b/src/postgres/discovery/mod.rs @@ -8,7 +8,7 @@ use crate::postgres::query::{ TableQueryResult, }; use futures::future; -use sea_query::{Alias, ColumnRef, Expr, Iden, IntoIden, JoinType, SeaRc, SelectStatement}; +use sea_query::{Alias, Iden, IntoIden, SeaRc}; use std::collections::HashMap; mod executor; diff --git a/src/postgres/query/enumeration.rs b/src/postgres/query/enumeration.rs index ac22ed8f..a1269c41 100644 --- a/src/postgres/query/enumeration.rs +++ b/src/postgres/query/enumeration.rs @@ -1,6 +1,6 @@ -use super::{InformationSchema, SchemaQueryBuilder}; +use super::SchemaQueryBuilder; use crate::sqlx_types::postgres::PgRow; -use sea_query::{Expr, Iden, Order, Query, SeaRc, SelectStatement}; +use sea_query::{Expr, Order, Query, SelectStatement}; #[derive(Debug, sea_query::Iden)] pub enum PgType {