From aee5bfcbc8b2581c54cefd5ae88d1f49c862ad9a Mon Sep 17 00:00:00 2001 From: Vincent Laugier Date: Sun, 3 Mar 2024 08:34:40 +0000 Subject: [PATCH 1/7] changes --- src/PostgresORM.jl | 4 +- src/SchemaInfo/SchemaInfo-imp.jl | 353 ---- src/SchemaInfo/{SchemaInfo-def.jl => _def.jl} | 1 + src/SchemaInfo/_imp.jl | 14 + src/SchemaInfo/analyse_db_schema.jl | 20 + ..._if_table_is_partition_of_another_table.jl | 23 + .../check_if_table_or_partition_exists.jl | 11 + src/SchemaInfo/get_columns_types.jl | 59 + src/SchemaInfo/get_custom_types.jl | 68 + src/SchemaInfo/get_fks.jl | 75 + src/SchemaInfo/get_pks.jl | 56 + src/SchemaInfo/get_schemas.jl | 11 + src/SchemaInfo/get_table_comment.jl | 21 + src/SchemaInfo/get_table_oid.jl | 31 + src/SchemaInfo/get_tables.jl | 17 + src/Tool/Tool-typescript.jl | 4 + src/Tool/Tool.jl | 1529 +++++++++-------- 17 files changed, 1225 insertions(+), 1072 deletions(-) delete mode 100644 src/SchemaInfo/SchemaInfo-imp.jl rename src/SchemaInfo/{SchemaInfo-def.jl => _def.jl} (91%) create mode 100644 src/SchemaInfo/_imp.jl create mode 100644 src/SchemaInfo/analyse_db_schema.jl create mode 100644 src/SchemaInfo/check_if_table_is_partition_of_another_table.jl create mode 100644 src/SchemaInfo/check_if_table_or_partition_exists.jl create mode 100644 src/SchemaInfo/get_columns_types.jl create mode 100644 src/SchemaInfo/get_custom_types.jl create mode 100644 src/SchemaInfo/get_fks.jl create mode 100644 src/SchemaInfo/get_pks.jl create mode 100644 src/SchemaInfo/get_schemas.jl create mode 100644 src/SchemaInfo/get_table_comment.jl create mode 100644 src/SchemaInfo/get_table_oid.jl create mode 100644 src/SchemaInfo/get_tables.jl diff --git a/src/PostgresORM.jl b/src/PostgresORM.jl index b4034ea..1432452 100644 --- a/src/PostgresORM.jl +++ b/src/PostgresORM.jl @@ -70,7 +70,7 @@ module PostgresORM # Provides functions to get information about the database structures module SchemaInfo - include("./SchemaInfo/SchemaInfo-def.jl") # This is only the definition of the + include("./SchemaInfo/_def.jl") # This is only the definition of the # module. See below for the actual # implementation. end #module SchemaInfo @@ -116,7 +116,7 @@ module PostgresORM end # module Tool # Implementation of the SchemaInfo module - include("./SchemaInfo/SchemaInfo-imp.jl") + include("./SchemaInfo/_imp.jl") include("./exposed-functions-from-submodules.jl") diff --git a/src/SchemaInfo/SchemaInfo-imp.jl b/src/SchemaInfo/SchemaInfo-imp.jl deleted file mode 100644 index 569e8fc..0000000 --- a/src/SchemaInfo/SchemaInfo-imp.jl +++ /dev/null @@ -1,353 +0,0 @@ -using LibPQ -using ..PostgresORMUtil, ..Controller - -function SchemaInfo.analyse_db_schema(dbconn::LibPQ.Connection) - - result = Dict() - - for schema in SchemaInfo.get_schemas(dbconn) - result[schema] = Dict() - for tbl in SchemaInfo.get_tables(schema, dbconn) - result[schema][tbl] = Dict( - :pk => SchemaInfo.get_pks(tbl, schema, dbconn), - :fks => SchemaInfo.get_fks(tbl, schema, dbconn), - :cols => SchemaInfo.get_columns_types(tbl,schema, dbconn), - :is_partition => SchemaInfo.check_if_table_is_partition_of_another_table(tbl, schema, dbconn) - ) - end # ENDOF for SchemaInfo.get_tables - end # ENDOF for schema - - return result - -end - -function SchemaInfo.get_schemas(dbconn::LibPQ.Connection) - - querystr = "SELECT n.nspname AS \"Name\", - pg_catalog.pg_get_userbyid(n.nspowner) AS \"Owner\" - FROM pg_catalog.pg_namespace n - WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' - ORDER BY 1;" - - queryres = execute_plain_query(querystr, [], dbconn) - return convert(Vector{String}, queryres.Name) -end - -function SchemaInfo.get_tables(schema::String, dbconn::LibPQ.Connection) - - querystr = "SELECT n.nspname as \"Schema\", - c.relname as \"Name\", - CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as \"Type\", - pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\" - FROM pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE c.relkind IN ('r','p','s','') - AND n.nspname !~ '^pg_toast' - AND n.nspname OPERATOR(pg_catalog.~) \$1 COLLATE pg_catalog.default - ORDER BY 1,2;" - println(querystr) - schema_pattern = "^($schema)\$" - queryres = execute_plain_query(querystr, [schema_pattern], dbconn) - return convert(Vector{String}, queryres.Name) - -end - -function SchemaInfo.check_if_table_is_partition_of_another_table( - table::String, - schema::String, - dbconn::LibPQ.Connection) - - # SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, c.relhastriggers, c.relrowsecurity, c.relforcerowsecurity, false AS relhasoids, c.relispartition, '', c.reltablespace, CASE WHEN c.reloftype = 0 THEN '' ELSE c.reloftype::pg_catalog.regtype::pg_catalog.text END, c.relpersistence, c.relreplident, am.amname - # FROM pg_catalog.pg_class c - # LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) - # LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) - # WHERE c.oid = '116102'; - - oid = SchemaInfo.get_table_oid(table, schema, dbconn) - querystr = "SELECT c.relispartition - FROM pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) - LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) - WHERE c.oid = \$1;" - - queryres = execute_plain_query(querystr, [oid], dbconn) - return queryres.relispartition[1] - -end - -function SchemaInfo.check_if_table_or_partition_exists(table::String, - schema::String, - dbconn::LibPQ.Connection) - querystr = "SELECT EXISTS ( - SELECT FROM information_schema.tables - WHERE table_name = \$1 - AND table_schema = \$2 - );" - queryres = execute_plain_query(querystr, [table, schema], dbconn) - return queryres[1,1] -end - -function SchemaInfo.get_table_oid(table::String, schema::String, dbconn::LibPQ.Connection) - - # pagila=# SELECT c.oid, - # pagila-# n.nspname, - # pagila-# c.relname - # pagila-# FROM pg_catalog.pg_class c - # pagila-# LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - # pagila-# WHERE c.relname OPERATOR(pg_catalog.~) '^(film_actor)$' COLLATE pg_catalog.default - # pagila-# AND pg_catalog.pg_table_is_visible(c.oid) - # pagila-# ORDER BY 2, 3; - # oid | nspname | relname - # --------+---------+------------ - # 106295 | public | film_actor - # (1 row) - - querystr = "SELECT c.oid, - n.nspname, - c.relname - FROM pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE c.relname OPERATOR(pg_catalog.~) \$1 COLLATE pg_catalog.default - AND n.nspname OPERATOR(pg_catalog.~) \$2 COLLATE pg_catalog.default - ORDER BY 2, 3;" - - table_pattern = "^($table)\$" - schema_pattern = "^($schema)\$" - - queryres = execute_plain_query(querystr, [table_pattern,schema_pattern], dbconn) - return signed(queryres.oid[1]) - -end - -function SchemaInfo.get_pks(table::String, schema::String, dbconn::LibPQ.Connection) - - # pagila=# SELECT c2.relname, i.indisprimary, i.indisunique, i.indisclustered, i.indisvalid, pg_catalog.pg_get_indexdef(i.indexrelid, 0, true), - # pagila-# pg_catalog.pg_get_constraintdef(con.oid, true), contype, condeferrable, condeferred, i.indisreplident, c2.reltablespace - # pagila-# FROM pg_catalog.pg_class c, pg_catalog.pg_class c2, pg_catalog.pg_index i - # pagila-# LEFT JOIN pg_catalog.pg_constraint con ON (conrelid = i.indrelid AND conindid = i.indexrelid AND contype IN ('p','u','x')) - # pagila-# WHERE c.oid = '106295' AND c.oid = i.indrelid AND i.indexrelid = c2.oid - # pagila-# ORDER BY i.indisprimary DESC, i.indisunique DESC, c2.relname; - # relname | indisprimary | indisunique | indisclustered | indisvalid | pg_get_indexdef | pg_get_constraintdef | contype | condeferrable | condeferred | i - # ndisreplident | reltablespace - # -----------------+--------------+-------------+----------------+------------+-----------------------------------------------------------------------------------+---------------------------------+---------+---------------+-------------+-- - # --------------+--------------- - # film_actor_pkey | t | t | f | t | CREATE UNIQUE INDEX film_actor_pkey ON film_actor USING btree (actor_id, film_id) | PRIMARY KEY (actor_id, film_id) | p | f | f | f - # | 0 - # idx_fk_film_id | f | f | f | t | CREATE INDEX idx_fk_film_id ON film_actor USING btree (film_id) | | | | | f - # | 0 - # (2 rows) - - - querystr = "SELECT c2.relname, i.indisprimary, i.indisunique, i.indisclustered, i.indisvalid, pg_catalog.pg_get_indexdef(i.indexrelid, 0, true), - pg_catalog.pg_get_constraintdef(con.oid, true), contype, condeferrable, condeferred, i.indisreplident, c2.reltablespace - FROM pg_catalog.pg_class c, pg_catalog.pg_class c2, pg_catalog.pg_index i - LEFT JOIN pg_catalog.pg_constraint con ON (conrelid = i.indrelid AND conindid = i.indexrelid AND contype IN ('p','u','x')) - WHERE c.oid = \$1 AND c.oid = i.indrelid AND i.indexrelid = c2.oid - ORDER BY i.indisprimary DESC, i.indisunique DESC, c2.relname;" - - table_oid = SchemaInfo.get_table_oid(table, schema, dbconn) - - queryres = execute_plain_query(querystr, [table_oid], dbconn) - pg_get_constraintdef = filter(x -> begin - !ismissing(x) && startswith(x,"PRIMARY KEY") - end, - queryres.pg_get_constraintdef) - - pg_get_constraintdef = convert(Vector{String}, pg_get_constraintdef) - # pg_get_constraintdef = map(x -> replace(x, " " => ""), pg_get_constraintdef) - # return pg_get_constraintdef - - result = Vector{String}() - for str in pg_get_constraintdef - pks = match(r"^PRIMARY KEY \((.*)\)$",str) - pks = convert(Vector{String}, pks.captures) - tmp_res = String[] - for pk in pks - push!(tmp_res, - remove_spaces_and_split(pk)...) - end - sort!(tmp_res) # IMPORTANT! Sort the PK columns, we'll do the same with - # the FK columns - push!(result, tmp_res...) - end - - return result - - -end - - -function SchemaInfo.get_fks(table::String, schema::String, dbconn::LibPQ.Connection) - - # pagila=# SELECT true as sametable, conname, - # pagila-# pg_catalog.pg_get_constraintdef(r.oid, true) as condef, - # pagila-# conrelid::pg_catalog.regclass AS ontable - # pagila-# FROM pg_catalog.pg_constraint r - # pagila-# WHERE r.conrelid = '106295' AND r.contype = 'f' - # pagila-# AND conparentid = 0 - # pagila-# ORDER BY conname - # pagila-# - # pagila-# ; - # sametable | conname | condef | ontable - # -----------+--------------------------+----------------------------------------------------------------------------------------+------------ - # t | film_actor_actor_id_fkey | FOREIGN KEY (actor_id) REFERENCES actor(actor_id) ON UPDATE CASCADE ON DELETE RESTRICT | film_actor - # t | film_actor_film_release_year_fkey | FOREIGN KEY (film_id, film_release_year) REFERENCES film(film_id, release_year) | film_actor - # (2 rows) - - querystr = "SELECT true as sametable, conname, - pg_catalog.pg_get_constraintdef(r.oid, true) as condef, - conrelid::pg_catalog.regclass AS ontable - FROM pg_catalog.pg_constraint r - WHERE r.conrelid = \$1 AND r.contype = 'f' - AND conparentid = 0 - ORDER BY conname" - - table_oid = SchemaInfo.get_table_oid(table, schema, dbconn) - - queryres = execute_plain_query(querystr, [table_oid], dbconn) - queryres = filter(x -> begin - !ismissing(x.condef) && startswith(x.condef,"FOREIGN KEY") - end, - queryres) - - result = Dict() - for r in eachrow(queryres) - fks = match(r"^FOREIGN KEY \((.*)\) REFERENCES (.*)\((.*)\)",r.condef) - # @info fks.captures - - # Referencing columns - referencing_cols = remove_spaces_and_split(string(fks.captures[1])) - referencing_cols = string.(referencing_cols) - - # Referenced table - referenced_table = remove_spaces_and_split(string(fks.captures[2]))[1] - referenced_schema = "public" - if occursin(".",referenced_table) - referenced_table_arr = split(referenced_table,'.') - referenced_table = referenced_table_arr[2] - referenced_schema = referenced_table_arr[1] - end - - # Referenced columns - referenced_cols = remove_spaces_and_split(string(fks.captures[3])) - referenced_cols = string.(referenced_cols) - - # Reorder the pairs (PK column, FK column) according to the PK columns - ordered_cols_along_pk_cols = - collect(zip(referenced_cols, referencing_cols)) |> - n -> sort(n, by = x -> x[1]) - referenced_cols = map(x -> x[1],ordered_cols_along_pk_cols) - referencing_cols = map(x -> x[2],ordered_cols_along_pk_cols) - - result[r.conname] = Dict( - :referencing_cols => referencing_cols, - :referenced_table => Dict(:table => string(referenced_table), - :schema => string(referenced_schema)), - :referenced_cols => referenced_cols - ) - - end - - - - return result - - -end - -function SchemaInfo.get_columns_types(table::String,schema::String, dbconn::LibPQ.Connection) - query_string = "SELECT column_name, - data_type AS column_type, - udt_name AS element_type - from information_schema.columns - WHERE table_schema = \$1 AND table_name = \$2" - - cols = execute_plain_query(query_string, - [schema,table], - dbconn) - - result = Dict() - - for c in eachrow(cols) - colname = c.column_name - coltype = c.column_type - elttype = c.element_type - result[colname] = Dict( - :type => coltype, - :elttype_if_array => elttype - ) - end - - return result - -end - - -function SchemaInfo.get_custom_types(dbconn::LibPQ.Connection) - - - # schema | name | internal_name | size | elements | description - # --------+------------+---------------+------+------------+------------- - # public | value_type | value_type | 4 | ordinal +| - # | | | | continuous+| - # | | | | category +| - # | | | | bool +| - # | | | | text | - - query_string = "SELECT n.nspname AS schema, - pg_catalog.format_type ( t.oid, NULL ) AS name, - t.typname AS internal_name, - CASE - WHEN t.typrelid != 0 - THEN CAST ( 'tuple' AS pg_catalog.text ) - WHEN t.typlen < 0 - THEN CAST ( 'var' AS pg_catalog.text ) - ELSE CAST ( t.typlen AS pg_catalog.text ) - END AS size, - pg_catalog.array_to_string ( - ARRAY( SELECT e.enumlabel - FROM pg_catalog.pg_enum e - WHERE e.enumtypid = t.oid - ORDER BY e.oid ), E'\n' - ) AS elements, - pg_catalog.obj_description ( t.oid, 'pg_type' ) AS description - FROM pg_catalog.pg_type t - LEFT JOIN pg_catalog.pg_namespace n - ON n.oid = t.typnamespace - WHERE ( t.typrelid = 0 - OR ( SELECT c.relkind = 'c' - FROM pg_catalog.pg_class c - WHERE c.oid = t.typrelid - ) - ) - AND NOT EXISTS - ( SELECT 1 - FROM pg_catalog.pg_type el - WHERE el.oid = t.typelem - AND el.typarray = t.oid - ) - AND n.nspname <> 'pg_catalog' - AND n.nspname <> 'information_schema' - AND pg_catalog.pg_type_is_visible ( t.oid ) - ORDER BY 1, 2;" - - cols = execute_plain_query(query_string, - [], - dbconn) - - result = Dict() - - for c in eachrow(cols) - name_ = c.internal_name - possible_values = string.(split(c.elements, "\n")) - # Cleaning - filter!(x-> length(x) > 0,possible_values) - result[name_] = Dict( - :name => name_, - :possible_values => possible_values - ) - end - - return result - -end diff --git a/src/SchemaInfo/SchemaInfo-def.jl b/src/SchemaInfo/_def.jl similarity index 91% rename from src/SchemaInfo/SchemaInfo-def.jl rename to src/SchemaInfo/_def.jl index f55997b..09d111e 100644 --- a/src/SchemaInfo/SchemaInfo-def.jl +++ b/src/SchemaInfo/_def.jl @@ -8,3 +8,4 @@ function analyse_db_schema end function check_if_table_is_partition_of_another_table end function check_if_table_or_partition_exists end function get_custom_types end +function get_table_comment end diff --git a/src/SchemaInfo/_imp.jl b/src/SchemaInfo/_imp.jl new file mode 100644 index 0000000..b84d459 --- /dev/null +++ b/src/SchemaInfo/_imp.jl @@ -0,0 +1,14 @@ +using LibPQ +using ..PostgresORMUtil, ..Controller + +include("analyse_db_schema.jl") +include("check_if_table_is_partition_of_another_table.jl") +include("check_if_table_or_partition_exists.jl") +include("get_columns_types.jl") +include("get_custom_types.jl") +include("get_fks.jl") +include("get_pks.jl") +include("get_schemas.jl") +include("get_table_oid.jl") +include("get_tables.jl") +include("get_table_comment.jl") diff --git a/src/SchemaInfo/analyse_db_schema.jl b/src/SchemaInfo/analyse_db_schema.jl new file mode 100644 index 0000000..0bf42ee --- /dev/null +++ b/src/SchemaInfo/analyse_db_schema.jl @@ -0,0 +1,20 @@ +function SchemaInfo.analyse_db_schema(dbconn::LibPQ.Connection) + + result = Dict() + + for schema in SchemaInfo.get_schemas(dbconn) + result[schema] = Dict() + for tbl in SchemaInfo.get_tables(schema, dbconn) + result[schema][tbl] = Dict( + :comment => SchemaInfo.get_table_comment(tbl, schema, dbconn), + :pk => SchemaInfo.get_pks(tbl, schema, dbconn), + :fks => SchemaInfo.get_fks(tbl, schema, dbconn), + :cols => SchemaInfo.get_columns_types(tbl,schema, dbconn), + :is_partition => SchemaInfo.check_if_table_is_partition_of_another_table(tbl, schema, dbconn) + ) + end # ENDOF for SchemaInfo.get_tables + end # ENDOF for schema + + return result + + end diff --git a/src/SchemaInfo/check_if_table_is_partition_of_another_table.jl b/src/SchemaInfo/check_if_table_is_partition_of_another_table.jl new file mode 100644 index 0000000..5a211be --- /dev/null +++ b/src/SchemaInfo/check_if_table_is_partition_of_another_table.jl @@ -0,0 +1,23 @@ +function SchemaInfo.check_if_table_is_partition_of_another_table( + table::String, + schema::String, + dbconn::LibPQ.Connection +) + +# SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, c.relhastriggers, c.relrowsecurity, c.relforcerowsecurity, false AS relhasoids, c.relispartition, '', c.reltablespace, CASE WHEN c.reloftype = 0 THEN '' ELSE c.reloftype::pg_catalog.regtype::pg_catalog.text END, c.relpersistence, c.relreplident, am.amname +# FROM pg_catalog.pg_class c +# LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) +# LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) +# WHERE c.oid = '116102'; + +oid = SchemaInfo.get_table_oid(table, schema, dbconn) +querystr = "SELECT c.relispartition +FROM pg_catalog.pg_class c +LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) +LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) +WHERE c.oid = \$1;" + +queryres = execute_plain_query(querystr, [oid], dbconn) +return queryres.relispartition[1] + +end diff --git a/src/SchemaInfo/check_if_table_or_partition_exists.jl b/src/SchemaInfo/check_if_table_or_partition_exists.jl new file mode 100644 index 0000000..83196b4 --- /dev/null +++ b/src/SchemaInfo/check_if_table_or_partition_exists.jl @@ -0,0 +1,11 @@ +function SchemaInfo.check_if_table_or_partition_exists(table::String, + schema::String, + dbconn::LibPQ.Connection) + querystr = "SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = \$1 + AND table_schema = \$2 + );" + queryres = execute_plain_query(querystr, [table, schema], dbconn) + return queryres[1,1] +end diff --git a/src/SchemaInfo/get_columns_types.jl b/src/SchemaInfo/get_columns_types.jl new file mode 100644 index 0000000..44301e1 --- /dev/null +++ b/src/SchemaInfo/get_columns_types.jl @@ -0,0 +1,59 @@ +function SchemaInfo.get_columns_types(table::String,schema::String, dbconn::LibPQ.Connection) + + query_string = " + SELECT + c.column_name, + c.data_type AS column_type, + c.udt_name AS element_type, + d.description AS column_comment + FROM + information_schema.columns c + LEFT JOIN + pg_description d ON d.objoid = ( + SELECT oid + FROM pg_class + WHERE relname = \$2 + AND relnamespace = ( + SELECT oid + FROM pg_namespace + WHERE nspname = \$1) + ) + AND d.objsubid = ( + SELECT attnum + FROM pg_attribute + WHERE attrelid = ( + SELECT oid + FROM pg_class + WHERE relname = \$2 + AND relnamespace = ( + SELECT oid + FROM pg_namespace + WHERE nspname = \$1 + ) + ) + AND attname = c.column_name + ) + WHERE c.table_schema = \$1 + AND c.table_name = \$2" + + cols = execute_plain_query(query_string, + [schema,table], + dbconn) + + result = Dict() + + for c in eachrow(cols) + colname = c.column_name + coltype = c.column_type + elttype = c.element_type + comment = c.column_comment + result[colname] = Dict( + :type => coltype, + :elttype_if_array => elttype, + :comment => comment + ) + end + + return result + +end diff --git a/src/SchemaInfo/get_custom_types.jl b/src/SchemaInfo/get_custom_types.jl new file mode 100644 index 0000000..88dbcb7 --- /dev/null +++ b/src/SchemaInfo/get_custom_types.jl @@ -0,0 +1,68 @@ +function SchemaInfo.get_custom_types(dbconn::LibPQ.Connection) + + + # schema | name | internal_name | size | elements | description + # --------+------------+---------------+------+------------+------------- + # public | value_type | value_type | 4 | ordinal +| + # | | | | continuous+| + # | | | | category +| + # | | | | bool +| + # | | | | text | + + query_string = "SELECT n.nspname AS schema, + pg_catalog.format_type ( t.oid, NULL ) AS name, + t.typname AS internal_name, + CASE + WHEN t.typrelid != 0 + THEN CAST ( 'tuple' AS pg_catalog.text ) + WHEN t.typlen < 0 + THEN CAST ( 'var' AS pg_catalog.text ) + ELSE CAST ( t.typlen AS pg_catalog.text ) + END AS size, + pg_catalog.array_to_string ( + ARRAY( SELECT e.enumlabel + FROM pg_catalog.pg_enum e + WHERE e.enumtypid = t.oid + ORDER BY e.oid ), E'\n' + ) AS elements, + pg_catalog.obj_description ( t.oid, 'pg_type' ) AS description + FROM pg_catalog.pg_type t + LEFT JOIN pg_catalog.pg_namespace n + ON n.oid = t.typnamespace + WHERE ( t.typrelid = 0 + OR ( SELECT c.relkind = 'c' + FROM pg_catalog.pg_class c + WHERE c.oid = t.typrelid + ) + ) + AND NOT EXISTS + ( SELECT 1 + FROM pg_catalog.pg_type el + WHERE el.oid = t.typelem + AND el.typarray = t.oid + ) + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND pg_catalog.pg_type_is_visible ( t.oid ) + ORDER BY 1, 2;" + + cols = execute_plain_query(query_string, + [], + dbconn) + + result = Dict() + + for c in eachrow(cols) + name_ = c.internal_name + possible_values = string.(split(c.elements, "\n")) + # Cleaning + filter!(x-> length(x) > 0,possible_values) + result[name_] = Dict( + :name => name_, + :possible_values => possible_values + ) + end + + return result + +end diff --git a/src/SchemaInfo/get_fks.jl b/src/SchemaInfo/get_fks.jl new file mode 100644 index 0000000..568f76c --- /dev/null +++ b/src/SchemaInfo/get_fks.jl @@ -0,0 +1,75 @@ + +function SchemaInfo.get_fks(table::String, schema::String, dbconn::LibPQ.Connection) + + # pagila=# SELECT true as sametable, conname, + # pagila-# pg_catalog.pg_get_constraintdef(r.oid, true) as condef, + # pagila-# conrelid::pg_catalog.regclass AS ontable + # pagila-# FROM pg_catalog.pg_constraint r + # pagila-# WHERE r.conrelid = '106295' AND r.contype = 'f' + # pagila-# AND conparentid = 0 + # pagila-# ORDER BY conname + # pagila-# + # pagila-# ; + # sametable | conname | condef | ontable + # -----------+--------------------------+----------------------------------------------------------------------------------------+------------ + # t | film_actor_actor_id_fkey | FOREIGN KEY (actor_id) REFERENCES actor(actor_id) ON UPDATE CASCADE ON DELETE RESTRICT | film_actor + # t | film_actor_film_release_year_fkey | FOREIGN KEY (film_id, film_release_year) REFERENCES film(film_id, release_year) | film_actor + # (2 rows) + + querystr = "SELECT true as sametable, conname, + pg_catalog.pg_get_constraintdef(r.oid, true) as condef, + conrelid::pg_catalog.regclass AS ontable + FROM pg_catalog.pg_constraint r + WHERE r.conrelid = \$1 AND r.contype = 'f' + AND conparentid = 0 + ORDER BY conname" + + table_oid = SchemaInfo.get_table_oid(table, schema, dbconn) + + queryres = execute_plain_query(querystr, [table_oid], dbconn) + queryres = filter(x -> begin + !ismissing(x.condef) && startswith(x.condef,"FOREIGN KEY") + end, + queryres) + + result = Dict() + for r in eachrow(queryres) + fks = match(r"^FOREIGN KEY \((.*)\) REFERENCES (.*)\((.*)\)",r.condef) + # @info fks.captures + + # Referencing columns + referencing_cols = remove_spaces_and_split(string(fks.captures[1])) + referencing_cols = string.(referencing_cols) + + # Referenced table + referenced_table = remove_spaces_and_split(string(fks.captures[2]))[1] + referenced_schema = "public" + if occursin(".",referenced_table) + referenced_table_arr = split(referenced_table,'.') + referenced_table = referenced_table_arr[2] + referenced_schema = referenced_table_arr[1] + end + + # Referenced columns + referenced_cols = remove_spaces_and_split(string(fks.captures[3])) + referenced_cols = string.(referenced_cols) + + # Reorder the pairs (PK column, FK column) according to the PK columns + ordered_cols_along_pk_cols = + collect(zip(referenced_cols, referencing_cols)) |> + n -> sort(n, by = x -> x[1]) + referenced_cols = map(x -> x[1],ordered_cols_along_pk_cols) + referencing_cols = map(x -> x[2],ordered_cols_along_pk_cols) + + result[r.conname] = Dict( + :referencing_cols => referencing_cols, + :referenced_table => Dict(:table => string(referenced_table), + :schema => string(referenced_schema)), + :referenced_cols => referenced_cols + ) + + end + + return result + + end diff --git a/src/SchemaInfo/get_pks.jl b/src/SchemaInfo/get_pks.jl new file mode 100644 index 0000000..fc168e4 --- /dev/null +++ b/src/SchemaInfo/get_pks.jl @@ -0,0 +1,56 @@ +function SchemaInfo.get_pks(table::String, schema::String, dbconn::LibPQ.Connection) + + # pagila=# SELECT c2.relname, i.indisprimary, i.indisunique, i.indisclustered, i.indisvalid, pg_catalog.pg_get_indexdef(i.indexrelid, 0, true), + # pagila-# pg_catalog.pg_get_constraintdef(con.oid, true), contype, condeferrable, condeferred, i.indisreplident, c2.reltablespace + # pagila-# FROM pg_catalog.pg_class c, pg_catalog.pg_class c2, pg_catalog.pg_index i + # pagila-# LEFT JOIN pg_catalog.pg_constraint con ON (conrelid = i.indrelid AND conindid = i.indexrelid AND contype IN ('p','u','x')) + # pagila-# WHERE c.oid = '106295' AND c.oid = i.indrelid AND i.indexrelid = c2.oid + # pagila-# ORDER BY i.indisprimary DESC, i.indisunique DESC, c2.relname; + # relname | indisprimary | indisunique | indisclustered | indisvalid | pg_get_indexdef | pg_get_constraintdef | contype | condeferrable | condeferred | i + # ndisreplident | reltablespace + # -----------------+--------------+-------------+----------------+------------+-----------------------------------------------------------------------------------+---------------------------------+---------+---------------+-------------+-- + # --------------+--------------- + # film_actor_pkey | t | t | f | t | CREATE UNIQUE INDEX film_actor_pkey ON film_actor USING btree (actor_id, film_id) | PRIMARY KEY (actor_id, film_id) | p | f | f | f + # | 0 + # idx_fk_film_id | f | f | f | t | CREATE INDEX idx_fk_film_id ON film_actor USING btree (film_id) | | | | | f + # | 0 + # (2 rows) + + + querystr = "SELECT c2.relname, i.indisprimary, i.indisunique, i.indisclustered, i.indisvalid, pg_catalog.pg_get_indexdef(i.indexrelid, 0, true), + pg_catalog.pg_get_constraintdef(con.oid, true), contype, condeferrable, condeferred, i.indisreplident, c2.reltablespace + FROM pg_catalog.pg_class c, pg_catalog.pg_class c2, pg_catalog.pg_index i + LEFT JOIN pg_catalog.pg_constraint con ON (conrelid = i.indrelid AND conindid = i.indexrelid AND contype IN ('p','u','x')) + WHERE c.oid = \$1 AND c.oid = i.indrelid AND i.indexrelid = c2.oid + ORDER BY i.indisprimary DESC, i.indisunique DESC, c2.relname;" + + table_oid = SchemaInfo.get_table_oid(table, schema, dbconn) + + queryres = execute_plain_query(querystr, [table_oid], dbconn) + pg_get_constraintdef = filter(x -> begin + !ismissing(x) && startswith(x,"PRIMARY KEY") + end, + queryres.pg_get_constraintdef) + + pg_get_constraintdef = convert(Vector{String}, pg_get_constraintdef) + # pg_get_constraintdef = map(x -> replace(x, " " => ""), pg_get_constraintdef) + # return pg_get_constraintdef + + result = Vector{String}() + for str in pg_get_constraintdef + pks = match(r"^PRIMARY KEY \((.*)\)$",str) + pks = convert(Vector{String}, pks.captures) + tmp_res = String[] + for pk in pks + push!(tmp_res, + remove_spaces_and_split(pk)...) + end + sort!(tmp_res) # IMPORTANT! Sort the PK columns, we'll do the same with + # the FK columns + push!(result, tmp_res...) + end + + return result + + +end diff --git a/src/SchemaInfo/get_schemas.jl b/src/SchemaInfo/get_schemas.jl new file mode 100644 index 0000000..1cab6ac --- /dev/null +++ b/src/SchemaInfo/get_schemas.jl @@ -0,0 +1,11 @@ +function SchemaInfo.get_schemas(dbconn::LibPQ.Connection) + + querystr = "SELECT n.nspname AS \"Name\", + pg_catalog.pg_get_userbyid(n.nspowner) AS \"Owner\" + FROM pg_catalog.pg_namespace n + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' + ORDER BY 1;" + + queryres = execute_plain_query(querystr, [], dbconn) + return convert(Vector{String}, queryres.Name) + end diff --git a/src/SchemaInfo/get_table_comment.jl b/src/SchemaInfo/get_table_comment.jl new file mode 100644 index 0000000..93a3832 --- /dev/null +++ b/src/SchemaInfo/get_table_comment.jl @@ -0,0 +1,21 @@ +function SchemaInfo.get_table_comment( + table::String, schema::String, dbconn::LibPQ.Connection +)::Union{Missing,String} + + querystr = " + SELECT + pg_catalog.obj_description(pg_class.oid, 'pg_class') AS table_comment + FROM + pg_catalog.pg_class + INNER JOIN + pg_catalog.pg_namespace ON pg_namespace.oid = pg_class.relnamespace + WHERE + pg_class.relname = \$2 + AND pg_namespace.nspname = \$1 + " + + queryres = execute_plain_query(querystr, [schema,table], dbconn) + + return queryres.table_comment[1] + +end diff --git a/src/SchemaInfo/get_table_oid.jl b/src/SchemaInfo/get_table_oid.jl new file mode 100644 index 0000000..9f78ea5 --- /dev/null +++ b/src/SchemaInfo/get_table_oid.jl @@ -0,0 +1,31 @@ +function SchemaInfo.get_table_oid(table::String, schema::String, dbconn::LibPQ.Connection) + + # pagila=# SELECT c.oid, + # pagila-# n.nspname, + # pagila-# c.relname + # pagila-# FROM pg_catalog.pg_class c + # pagila-# LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + # pagila-# WHERE c.relname OPERATOR(pg_catalog.~) '^(film_actor)$' COLLATE pg_catalog.default + # pagila-# AND pg_catalog.pg_table_is_visible(c.oid) + # pagila-# ORDER BY 2, 3; + # oid | nspname | relname + # --------+---------+------------ + # 106295 | public | film_actor + # (1 row) + + querystr = "SELECT c.oid, + n.nspname, + c.relname + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname OPERATOR(pg_catalog.~) \$1 COLLATE pg_catalog.default + AND n.nspname OPERATOR(pg_catalog.~) \$2 COLLATE pg_catalog.default + ORDER BY 2, 3;" + + table_pattern = "^($table)\$" + schema_pattern = "^($schema)\$" + + queryres = execute_plain_query(querystr, [table_pattern,schema_pattern], dbconn) + return signed(queryres.oid[1]) + + end diff --git a/src/SchemaInfo/get_tables.jl b/src/SchemaInfo/get_tables.jl new file mode 100644 index 0000000..a7606f5 --- /dev/null +++ b/src/SchemaInfo/get_tables.jl @@ -0,0 +1,17 @@ +function SchemaInfo.get_tables(schema::String, dbconn::LibPQ.Connection) + + querystr = "SELECT n.nspname as \"Schema\", + c.relname as \"Name\", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as \"Type\", + pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','p','s','') + AND n.nspname !~ '^pg_toast' + AND n.nspname OPERATOR(pg_catalog.~) \$1 COLLATE pg_catalog.default + ORDER BY 1,2;" + schema_pattern = "^($schema)\$" + queryres = execute_plain_query(querystr, [schema_pattern], dbconn) + return convert(Vector{String}, queryres.Name) + + end diff --git a/src/Tool/Tool-typescript.jl b/src/Tool/Tool-typescript.jl index 871572d..9f8bcbd 100644 --- a/src/Tool/Tool-typescript.jl +++ b/src/Tool/Tool-typescript.jl @@ -4,6 +4,8 @@ function generate_typescript_code(dbconn::LibPQ.Connection, ;lang_code = "eng", module_name_for_all_schemas::Union{String,Missing} = "Model") + @info "BEGIN Typescript code generation" + object_model = generate_object_model( dbconn, lang_code, @@ -16,6 +18,8 @@ function generate_typescript_code(dbconn::LibPQ.Connection, outdir, relative_path_to_enum_dir) + @info "ENDOF Typescript code generation" + end diff --git a/src/Tool/Tool.jl b/src/Tool/Tool.jl index 88f74f4..22b8847 100644 --- a/src/Tool/Tool.jl +++ b/src/Tool/Tool.jl @@ -1,11 +1,10 @@ function remove_id_from_name(str) if startswith(str,"id_") - return str[4:length(str)] + return str[4:length(str)] end - if endswith(str,"_id") - return str[1:length(str)-3] + return str[1:length(str)-3] end return str @@ -13,883 +12,979 @@ function remove_id_from_name(str) end function build_module_name(str::String - ;module_name_for_all_schemas::Union{String,Missing} = missing) - if ismissing(module_name_for_all_schemas) - return StringCases.classify(str) - else - return module_name_for_all_schemas - end + ;module_name_for_all_schemas::Union{String,Missing} = missing) + if ismissing(module_name_for_all_schemas) + return StringCases.classify(str) + else + return module_name_for_all_schemas + end end function build_enum_module_name(str::String) - StringCases.classify(str) + StringCases.classify(str) end function build_enum_type_name(str::String) - uppercase(StringCases.snakecase(str)) + uppercase(StringCases.snakecase(str)) end function build_enum_name_w_module(str::String) - string(build_enum_module_name(str), - ".", - build_enum_type_name(str)) + string(build_enum_module_name(str), + ".", + build_enum_type_name(str)) end function build_enum_value(str::String) - StringCases.snakecase(str) + StringCases.snakecase(str) end function build_struct_name(str::String) - StringCases.classify(str) + StringCases.classify(str) end function build_struct_abstract_name(str::String) - return "I$(build_struct_name(str))" + return "I$(build_struct_name(str))" end function build_field_name(str::String, - lang_code::String - ;replace_ids = false, - is_onetomany = false) - return build_field_name(tovector(str), - lang_code::String - ;replace_ids = replace_ids, - is_onetomany = is_onetomany) + lang_code::String + ;replace_ids = false, + is_onetomany = false) + return build_field_name(tovector(str), + lang_code::String + ;replace_ids = replace_ids, + is_onetomany = is_onetomany) end function build_field_name(str_arr::Vector{String}, - lang_code::String - ;replace_ids = false, - is_onetomany = false) - - if (replace_ids) - str_arr = map(x -> remove_id_from_name(x), - str_arr) - end - - field_name = join(str_arr, '_') - field_name = StringCases.camelize(field_name) - if is_onetomany - field_name = PostgresORMUtil.pluralize(field_name,lang_code) - end - - return field_name + lang_code::String + ;replace_ids = false, + is_onetomany = false) + + if (replace_ids) + str_arr = map(x -> remove_id_from_name(x), + str_arr) + end + + field_name = join(str_arr, '_') + field_name = StringCases.camelize(field_name) + if is_onetomany + field_name = PostgresORMUtil.pluralize(field_name,lang_code) + end + + return field_name end function is_vector_of_enum(coltype::String, - elttype::String, - customtypes_names::Vector{String}) - if (coltype == "ARRAY") - if elttype[2:end] in customtypes_names # remove the leading underscore - return true - end - end - return false + elttype::String, + customtypes_names::Vector{String}) + if (coltype == "ARRAY") + if elttype[2:end] in customtypes_names # remove the leading underscore + return true + end + end + return false end function is_vector_of_enum(coltype::String, - elttype::String, - customtypes::Dict) - customtypes_names = keys(customtypes) |> collect |> n -> string.(n) - return is_vector_of_enum(coltype, - elttype, - customtypes_names) + elttype::String, + customtypes::Dict) + customtypes_names = keys(customtypes) |> collect |> n -> string.(n) + return is_vector_of_enum(coltype, + elttype, + customtypes_names) end function get_fieldtype_from_coltype(coltype::String, - elttype::String, - customtypes::Dict, - ;tablename::String = "", - colname::String = "") - - attrtype = missing - customtypes_names = keys(customtypes) |> collect |> n -> string.(n) - - if (coltype == "character" - || coltype == "character varying" - || coltype == "text" - || coltype == "uuid") - attrtype = "String" - elseif (coltype == "boolean") - attrtype = "Bool" - elseif (coltype == "smallint") - attrtype = "Int16" - elseif (coltype == "integer" || coltype == "interval") - attrtype = "Int32" - elseif (coltype == "bigint") - attrtype = "Int64" - elseif (coltype == "numeric") - attrtype = "Float64" - elseif (coltype == "bytea") - attrtype = "Vector{UInt8}" - elseif (coltype == "date") - attrtype = "Date" - elseif (coltype == "time without time zone") - attrtype = "Time" - elseif (coltype == "timestamp without time zone") - attrtype = "DateTime" - elseif (coltype == "timestamp with time zone") - attrtype = "ZonedDateTime" - elseif (coltype == "ARRAY") - if (elttype == "_text" || elttype == "_varchar") - attrtype = "Vector{String}" - elseif (elttype == "_numeric") - attrtype = "Vector{Float64}" - elseif (elttype == "_int4") - attrtype = "Vector{Int64}" - elseif is_vector_of_enum(coltype,elttype,customtypes_names) - elttype = elttype[2:end] # remove the leading underscore - attrtype = "Vector{$(build_enum_name_w_module(elttype))}" - else - error("Unknown array type[$elttype] for table[$tablename] column[$colname]") - end - elseif (coltype == "tsvector") - attrtype = missing - elseif (coltype == "bytea") - attrtype = missing - elseif (coltype == "USER-DEFINED") - attrtype = build_enum_name_w_module(elttype) - else - error("Unknown type[$coltype] for table[$tablename] column[$colname]") - end + elttype::String, + customtypes::Dict, + ;tablename::String = "", + colname::String = "") - return attrtype -end + attrtype = missing + customtypes_names = keys(customtypes) |> collect |> n -> string.(n) -function generate_julia_code(dbconn::LibPQ.Connection, - outdir::String - ;lang_code = "eng", - module_name_for_all_schemas::Union{String,Missing} = "Model") - object_model = - generate_object_model(dbconn, - lang_code, - module_name_for_all_schemas = module_name_for_all_schemas) - generate_structs_from_object_model(object_model, outdir) - generate_orms_from_object_model(object_model, outdir) - generate_enums_from_object_model(object_model, outdir) + if (coltype == "character" + || coltype == "character varying" + || coltype == "text" + || coltype == "uuid") + attrtype = "String" + elseif (coltype == "boolean") + attrtype = "Bool" + elseif (coltype == "smallint") + attrtype = "Int16" + elseif (coltype == "integer" || coltype == "interval") + attrtype = "Int32" + elseif (coltype == "bigint") + attrtype = "Int64" + elseif (coltype == "numeric") + attrtype = "Float64" + elseif (coltype == "bytea") + attrtype = "Vector{UInt8}" + elseif (coltype == "date") + attrtype = "Date" + elseif (coltype == "time without time zone") + attrtype = "Time" + elseif (coltype == "timestamp without time zone") + attrtype = "DateTime" + elseif (coltype == "timestamp with time zone") + attrtype = "ZonedDateTime" + elseif (coltype == "ARRAY") + if (elttype == "_text" || elttype == "_varchar") + attrtype = "Vector{String}" + elseif (elttype == "_numeric") + attrtype = "Vector{Float64}" + elseif (elttype == "_int4") + attrtype = "Vector{Int64}" + elseif is_vector_of_enum(coltype,elttype,customtypes_names) + elttype = elttype[2:end] # remove the leading underscore + attrtype = "Vector{$(build_enum_name_w_module(elttype))}" + else + error("Unknown array type[$elttype] for table[$tablename] column[$colname]") + end + elseif (coltype == "tsvector") + attrtype = missing + elseif (coltype == "bytea") + attrtype = missing + elseif (coltype == "USER-DEFINED") + attrtype = build_enum_name_w_module(elttype) + else + error("Unknown type[$coltype] for table[$tablename] column[$colname]") + end + return attrtype end +function generate_julia_code( + dbconn::LibPQ.Connection, + outdir::String + ;lang_code = "eng", + module_name_for_all_schemas::Union{String,Missing} = "Model", + with_comment = true +) + + @info "BEGIN Julia code generation" + object_model = + generate_object_model(dbconn, + lang_code, + module_name_for_all_schemas = module_name_for_all_schemas) + generate_structs_from_object_model(object_model, outdir; with_comment = with_comment) + generate_orms_from_object_model(object_model, outdir) + generate_enums_from_object_model(object_model, outdir) + + @info "ENDOF Julia code generation" -function generate_object_model( - dbconn::LibPQ.Connection, - lang_code::String - ;ignored_columns::Vector{String} = Vector{String}(), - camelcase_is_default::Bool = true, - exceptions_to_default::Vector{String} = Vector{String}(), - module_name_for_all_schemas::Union{String,Missing} = missing) - - db_analysis = SchemaInfo.analyse_db_schema(dbconn) - custom_types = SchemaInfo.get_custom_types(dbconn) - - # Initialize result as Dict - object_model = Dict() - - # Initialize the various components of the result - modules = [] - structs = [] - fields = [] - enums = [] - - # Deal with the custom types - for (k,v) in custom_types - enum_name_w_module = build_enum_name_w_module(k) - enum_values = [] - for val in v[:possible_values] - enum_value = build_enum_value(val) - push!(enum_values, enum_value) - end - push!(enums, Dict(:module_name => build_enum_module_name(k), - :type_name => build_enum_type_name(k), - :values => enum_values)) - end # ENDOF for (k,v) in custom_types +end - # Deal with the schemas in ordre to fill in modules, structs and fields - for (schema, schemadef) in db_analysis - module_name = if ismissing(module_name_for_all_schemas) - build_module_name(schema - ;module_name_for_all_schemas = module_name_for_all_schemas) +function generate_object_model( + dbconn::LibPQ.Connection, + lang_code::String + ;ignored_columns::Vector{String} = Vector{String}(), + camelcase_is_default::Bool = true, + exceptions_to_default::Vector{String} = Vector{String}(), + module_name_for_all_schemas::Union{String,Missing} = missing +) + + db_analysis = SchemaInfo.analyse_db_schema(dbconn) + custom_types = SchemaInfo.get_custom_types(dbconn) + + # Initialize result as Dict + object_model = Dict() + + # Initialize the various components of the result + modules = [] + structs = [] + fields = [] + enums = [] + + # Deal with the custom types + for (k,v) in custom_types + enum_name_w_module = build_enum_name_w_module(k) + enum_values = [] + for val in v[:possible_values] + enum_value = build_enum_value(val) + push!(enum_values, enum_value) + end + push!(enums, Dict(:module_name => build_enum_module_name(k), + :type_name => build_enum_type_name(k), + :values => enum_values)) + end # ENDOF for (k,v) in custom_types + + # Deal with the schemas in ordre to fill in modules, structs and fields + for (schema, schemadef) in db_analysis + + module_name = if ismissing(module_name_for_all_schemas) + build_module_name(schema + ;module_name_for_all_schemas = module_name_for_all_schemas) + else + module_name_for_all_schemas + end + + _module = Dict(:name => module_name, + :schema => schema) + push!(modules, _module) + + for (table,tabledef) in schemadef + # Ignore table partitions (the partitioned table is the one of interest) + if tabledef[:is_partition] + continue + end + struct_name = build_struct_name(table) + struct_abstract_name = build_struct_abstract_name(table) + _struct = Dict( + :name => struct_name, + :module => _module, + :abstract_name => struct_abstract_name, + :table => table, + :schema => schema, + :comment => tabledef[:comment] + ) + push!(structs, _struct) + + # Initialize some temporary arrays for the different types of fields + struct_manytoone_fields = [] + struct_id_fields = [] + struct_basic_fields = [] + + # Loop over the FKs of the table to add the complex field + for (fkname,fkdef) in tabledef[:fks] + manytoone_field = Dict() + manytoone_field[:struct] = _struct + manytoone_field[:referenced_table] = fkdef[:referenced_table] + manytoone_field[:referenced_cols] = fkdef[:referenced_cols] + manytoone_field[:cols] = fkdef[:referencing_cols] + manytoone_field[:is_id] = false + if occursin("onetoone",fkname) + manytoone_field[:is_onetoone] = true + manytoone_field[:is_manytoone] = false + else + manytoone_field[:is_onetoone] = false + manytoone_field[:is_manytoone] = true + end + manytoone_field[:is_onetomany] = false + manytoone_field[:is_enum] = false + manytoone_field[:is_vectorofenum] = false + manytoone_field[:comment] = missing + + # Build a field name by one of the following options: + # Case 1: The FK is composed of one column only. In this case we use + # the name of the column. + # Case 2.1: The FK is composed of several columns and it is the only + # FK in this table pointing to the given target table. + # In this case we use the name of the target table. + # case 2.2 The FK is composed of several columns and it there are + # several FKs in this table pointing to the given target + # table. In this case we concatenate the referencing columns + nb_fks_same_table_same_targeted_table = tabledef[:fks] |> + n -> filter(x -> x[2][:referenced_table] == + manytoone_field[:referenced_table], + n) |> length + manytoone_field[:name] = if length(manytoone_field[:cols]) == 1 + build_field_name(manytoone_field[:cols], + lang_code, + ;replace_ids = true) else - module_name_for_all_schemas + if nb_fks_same_table_same_targeted_table == 1 + build_field_name(manytoone_field[:referenced_table][:table], + lang_code) + else + build_field_name(manytoone_field[:cols], + lang_code + ;replace_ids = true) + end end - _module = Dict(:name => module_name, - :schema => schema) - push!(modules, _module) + # Build the referenced module.struct name + # NOTE: We cannot get the information from 'structs' because the + # struct of interest is probably not available yet + + struct_name_w_module = ( + build_module_name(manytoone_field[:referenced_table][:schema] + ;module_name_for_all_schemas = module_name_for_all_schemas) + * "." + * build_struct_name(manytoone_field[:referenced_table][:table]) + ) + + struct_abstract_name_w_module = ( + build_module_name(manytoone_field[:referenced_table][:schema] + ;module_name_for_all_schemas = module_name_for_all_schemas) + * "." + * build_struct_abstract_name(manytoone_field[:referenced_table][:table]) + ) + + manytoone_field[:field_type] = struct_name_w_module + manytoone_field[:field_abstract_type] = struct_abstract_name_w_module + + push!(struct_manytoone_fields, manytoone_field) + + # Check whether all the referencing columns are in the PK columns, if + # yes also add the property to the id property of the struct + if all(map(x -> x in tabledef[:pk], + manytoone_field[:cols])) + manytoone_field[:is_id] = true + end + + end # for (fkname,fkdef) in tabledef[:fks] + + # Store the referencing columns for control + referencing_cols = [] + for v in (collect(values(tabledef[:fks])) |> + vect -> map(x -> x[:referencing_cols],vect)) + push!(referencing_cols,v...) + end - for (table,tabledef) in schemadef - # Ignore table partitions (the partitioned table is the one of interest) - if tabledef[:is_partition] - continue - end - struct_name = build_struct_name(table) - struct_abstract_name = build_struct_abstract_name(table) - _struct = Dict(:name => struct_name, - :module => _module, - :abstract_name => struct_abstract_name, - :table => table, - :schema => schema) - push!(structs, _struct) - - # Initialize some temporary arrays for the different types of fields - struct_manytoone_fields = [] - struct_id_fields = [] - struct_basic_fields = [] - - # Loop over the FKs of the table to add the complex field - for (fkname,fkdef) in tabledef[:fks] - manytoone_field = Dict() - manytoone_field[:struct] = _struct - manytoone_field[:referenced_table] = fkdef[:referenced_table] - manytoone_field[:referenced_cols] = fkdef[:referenced_cols] - manytoone_field[:cols] = fkdef[:referencing_cols] - manytoone_field[:is_id] = false - if occursin("onetoone",fkname) - manytoone_field[:is_onetoone] = true - manytoone_field[:is_manytoone] = false + # Loop over the PKs of the table, if the PK is found in a FKs, skip + # because we already added it when looping over the FKs + if tabledef[:pk] |> n -> map(x -> x ∈ referencing_cols, n) |> all + @info "PK of table[$table] is contained in a FK" else - manytoone_field[:is_onetoone] = false - manytoone_field[:is_manytoone] = true - end - manytoone_field[:is_onetomany] = false - manytoone_field[:is_enum] = false - manytoone_field[:is_vectorofenum] = false - - # Build a field name by one of the following options: - # Case 1: The FK is composed of one column only. In this case we use - # the name of the column. - # Case 2.1: The FK is composed of several columns and it is the only - # FK in this table pointing to the given target table. - # In this case we use the name of the target table. - # case 2.2 The FK is composed of several columns and it there are - # several FKs in this table pointing to the given target - # table. In this case we concatenate the referencing columns - nb_fks_same_table_same_targeted_table = tabledef[:fks] |> - n -> filter(x -> x[2][:referenced_table] == - manytoone_field[:referenced_table], - n) |> length - manytoone_field[:name] = if length(manytoone_field[:cols]) == 1 - build_field_name(manytoone_field[:cols], - lang_code, - ;replace_ids = true) - else - if nb_fks_same_table_same_targeted_table == 1 - build_field_name(manytoone_field[:referenced_table][:table], - lang_code) - else - build_field_name(manytoone_field[:cols], - lang_code - ;replace_ids = true) - end - end - - # Build the referenced module.struct name - # NOTE: We cannot get the information from 'structs' because the - # struct of interest is probably not available yet - - struct_name_w_module = ( - build_module_name(manytoone_field[:referenced_table][:schema] - ;module_name_for_all_schemas = module_name_for_all_schemas) - * "." - * build_struct_name(manytoone_field[:referenced_table][:table]) - ) - struct_abstract_name_w_module = ( - build_module_name(manytoone_field[:referenced_table][:schema] - ;module_name_for_all_schemas = module_name_for_all_schemas) - * "." - * build_struct_abstract_name(manytoone_field[:referenced_table][:table]) - ) + for pkcol in tabledef[:pk] + + id_field = Dict() + id_field[:struct] = _struct + id_field[:cols] = tovector(pkcol) + id_field[:is_id] = true + id_field[:is_manytoone] = false + id_field[:is_onetoone] = false + id_field[:is_onetomany] = false + id_field[:is_enum] = false + id_field[:is_vectorofenum] = false + field_name = build_field_name(pkcol,lang_code) + id_field[:name] = field_name + id_field[:comment] = missing + + field_type = + get_fieldtype_from_coltype(tabledef[:cols][pkcol][:type], + tabledef[:cols][pkcol][:elttype_if_array], + custom_types) + + # Check if it is an enum + if tabledef[:cols][pkcol][:type] == "USER-DEFINED" + id_field[:is_enum] = true + end - manytoone_field[:field_type] = struct_name_w_module - manytoone_field[:field_abstract_type] = struct_abstract_name_w_module + id_field[:field_type] = field_type + push!(struct_id_fields, id_field) - push!(struct_manytoone_fields, manytoone_field) + end # ENDOF for pk in tabledef[:pk] - # Check whether all the referencing columns are in the PK columns, if - # yes also add the property to the id property of the struct - if all(map(x -> x in tabledef[:pk], - manytoone_field[:cols])) - manytoone_field[:is_id] = true - end + end # ENDOF if tabledef[:pk] |> n -> map(x -> x ∈ referencing_cols, n) |> all - end # for (fkname,fkdef) in tabledef[:fks] - # Store the referencing columns for control - referencing_cols = [] - for v in (collect(values(tabledef[:fks])) |> - vect -> map(x -> x[:referencing_cols],vect)) - push!(referencing_cols,v...) - end - # Loop over the PKs of the table, if the pk is found in a FKs, skip - # because we already added it when looping over the FKs - for pkcol in tabledef[:pk] - - # Skip if we already handled this column - if pkcol in referencing_cols continue end - - id_field = Dict() - id_field[:struct] = _struct - id_field[:cols] = tovector(pkcol) - id_field[:is_id] = true - id_field[:is_manytoone] = false - id_field[:is_onetoone] = false - id_field[:is_onetomany] = false - id_field[:is_enum] = false - id_field[:is_vectorofenum] = false - field_name = build_field_name(pkcol,lang_code) - id_field[:name] = field_name - - field_type = - get_fieldtype_from_coltype(tabledef[:cols][pkcol][:type], - tabledef[:cols][pkcol][:elttype_if_array], - custom_types) - - # Check if it is an enum - if tabledef[:cols][pkcol][:type] == "USER-DEFINED" - id_field[:is_enum] = true + # Loop over all the columns of the table and skip the ones that have + # already been mapped to complex properties or id properties + for (colname, coldef) in tabledef[:cols] + if colname in referencing_cols continue end + if colname in tabledef[:pk] continue end + + basic_field = Dict() + basic_field[:struct] = _struct + basic_field[:cols] = tovector(colname) + basic_field[:is_id] = false + basic_field[:is_manytoone] = false + basic_field[:is_onetoone] = false + basic_field[:is_onetomany] = false + basic_field[:is_enum] = false + basic_field[:is_vectorofenum] = false + field_name = build_field_name(colname,lang_code) + basic_field[:name] = field_name + basic_field[:comment] = coldef[:comment] + + field_type = + get_fieldtype_from_coltype(coldef[:type], + coldef[:elttype_if_array], + custom_types + ;tablename = table, colname = colname) + + # Check if it is an enum or a vector of enum + if coldef[:type] == "USER-DEFINED" + basic_field[:is_enum] = true + elseif is_vector_of_enum(coldef[:type], coldef[:elttype_if_array], custom_types) + basic_field[:is_vectorofenum] = true + end + + basic_field[:field_type] = field_type + push!(struct_basic_fields, basic_field) + end - id_field[:field_type] = field_type - push!(struct_id_fields, id_field) - end # for pk in tabledef[:pk] - - - # Loop over all the columns of the table and skip the ones that have - # already been mapped to complex properties or id properties - for (colname, coldef) in tabledef[:cols] - if colname in referencing_cols continue end - if colname in tabledef[:pk] continue end - - basic_field = Dict() - basic_field[:struct] = _struct - basic_field[:cols] = tovector(colname) - basic_field[:is_id] = false - basic_field[:is_manytoone] = false - basic_field[:is_onetoone] = false - basic_field[:is_onetomany] = false - basic_field[:is_enum] = false - basic_field[:is_vectorofenum] = false - field_name = build_field_name(colname,lang_code) - basic_field[:name] = field_name - - field_type = - get_fieldtype_from_coltype(coldef[:type], - coldef[:elttype_if_array], - custom_types - ;tablename = table, colname = colname) - - # Check if it is an enum or a vector of enum - if coldef[:type] == "USER-DEFINED" - basic_field[:is_enum] = true - elseif is_vector_of_enum(coldef[:type], - coldef[:elttype_if_array], - custom_types) - basic_field[:is_vectorofenum] = true + # Enrich the arrays for all structs with this struct + push!(fields, struct_manytoone_fields...) + push!(fields, struct_id_fields...) + push!(fields, struct_basic_fields...) + + end # ENDOF for (table,tabledef) in schemadef + + end # ENDOF for (schema, schemadef) in db_analysis + + + # Now that we have all the struct known we can loop though the manytoone + # fieds and build the corresponding onetomany fields. + # Eg. For the manytoone (complex prop) 'Rental.staff' we build the onetomany + # 'Staff.rentals of type Vector{IRental}' + for manytoone_field in filter(x -> x[:is_manytoone] == true, fields) + onetomany_field = Dict() + onetomany_field[:is_id] = false + onetomany_field[:is_manytoone] = false + onetomany_field[:is_onetoone] = false + onetomany_field[:is_onetomany] = true + onetomany_field[:is_enum] = false + onetomany_field[:is_vectorofenum] = false + onetomany_type_name_w_module = manytoone_field[:field_type] # Public.Staff + onetomany_field[:comment] = missing + + # Build a field_name using one of the following options: + # Case1: The class holding the manytoone field has only one manytoone field + # pointing to the target class. In that case the name of the + # onetomany field in the target class will simply be the plural + # of the name of the manytoone field. + # Eg. Rental.staff -> rentals + # Case2: There are several manytoone fields in the class pointing to the + # the same target class. In that case we concatenate the manytoone + # field name and the onetoname table name and we make it plural + # Eg. Sentence.judge => CivilServant.judgeSentences + # Sentence.executionner => CivilServant.executionnerSentences + # + nb_manytoone_same_class_and_same_target_class = fields |> + n -> filter(x -> (x[:struct][:name] == manytoone_field[:struct][:name]),n) |> + n -> filter(x -> x[:is_manytoone],n) |> + n -> filter(x -> x[:field_type] == manytoone_field[:field_type],n) |> + length + + onetomany_field_name = if nb_manytoone_same_class_and_same_target_class == 1 + manytoone_field[:struct][:table] + else + string(manytoone_field[:name],"_",manytoone_field[:struct][:table]) end + onetomany_field_name = build_field_name(onetomany_field_name, + lang_code + ;is_onetomany = true) + onetomany_field_type = + "Vector{$(manytoone_field[:struct][:module][:name]).$(manytoone_field[:struct][:name])}" + onetomany_field_abstract_type = + "Vector{$(manytoone_field[:struct][:module][:name]).$(manytoone_field[:struct][:abstract_name])}" + + onetomany_field[:name] = onetomany_field_name + onetomany_field[:field_type] = onetomany_field_type + onetomany_field[:field_abstract_type] = onetomany_field_abstract_type + onetomany_field[:manytoone_field] = manytoone_field + + # Loop over the structs to link it to the + for _struct in structs + if "$(_struct[:module][:name]).$(_struct[:name])" == onetomany_type_name_w_module + onetomany_field[:struct] = _struct + end + end + if !haskey(onetomany_field,:struct) + error("Unable to find a struct $onetomany_type_name_w_module") + end - basic_field[:field_type] = field_type - push!(struct_basic_fields, basic_field) + # Add the fields to the other fields + # NOTE: There is no risk to creaet an infinite loop because we loop over + # the filtered array and not the array itself + push!(fields,onetomany_field) + end - end - # Enrich the arrays for all structs with this struct - push!(fields, struct_manytoone_fields...) - push!(fields, struct_id_fields...) - push!(fields, struct_basic_fields...) - - end # ENDOF for (table,tabledef) in schemadef - - end # ENDOF for (schema, schemadef) in db_analysis - - - # Now that we have all the struct known we can loop though the manytoone - # fieds and build the corresponding onetomany fields. - # Eg. For the manytoone (complex prop) 'Rental.staff' we build the onetomany - # 'Staff.rentals of type Vector{IRental}' - for manytoone_field in filter(x -> x[:is_manytoone] == true, fields) - onetomany_field = Dict() - onetomany_field[:is_id] = false - onetomany_field[:is_manytoone] = false - onetomany_field[:is_onetoone] = false - onetomany_field[:is_onetomany] = true - onetomany_field[:is_enum] = false - onetomany_field[:is_vectorofenum] = false - onetomany_type_name_w_module = manytoone_field[:field_type] # Public.Staff - - # Build a field_name using one of the following options: - # Case1: The class holding the manytoone field has only one manytoone field - # pointing to the target class. In that case the name of the - # onetomany field in the target class will simply be the plural - # of the name of the manytoone field. - # Eg. Rental.staff -> rentals - # Case2: There are several manytoone fields in the class pointing to the - # the same target class. In that case we concatenate the manytoone - # field name and the onetoname table name and we make it plural - # Eg. Sentence.judge => CivilServant.judgeSentences - # Sentence.executionner => CivilServant.executionnerSentences - # - nb_manytoone_same_class_and_same_target_class = fields |> - n -> filter(x -> (x[:struct][:name] == manytoone_field[:struct][:name]),n) |> - n -> filter(x -> x[:is_manytoone],n) |> - n -> filter(x -> x[:field_type] == manytoone_field[:field_type],n) |> - length - - onetomany_field_name = if nb_manytoone_same_class_and_same_target_class == 1 - manytoone_field[:struct][:table] - else - string(manytoone_field[:name],"_",manytoone_field[:struct][:table]) - end - onetomany_field_name = build_field_name(onetomany_field_name, - lang_code - ;is_onetomany = true) - onetomany_field_type = - "Vector{$(manytoone_field[:struct][:module][:name]).$(manytoone_field[:struct][:name])}" - onetomany_field_abstract_type = - "Vector{$(manytoone_field[:struct][:module][:name]).$(manytoone_field[:struct][:abstract_name])}" - - onetomany_field[:name] = onetomany_field_name - onetomany_field[:field_type] = onetomany_field_type - onetomany_field[:field_abstract_type] = onetomany_field_abstract_type - onetomany_field[:manytoone_field] = manytoone_field - - # Loop over the structs to link it to the - for _struct in structs - if "$(_struct[:module][:name]).$(_struct[:name])" == onetomany_type_name_w_module - onetomany_field[:struct] = _struct - end - end - if !haskey(onetomany_field,:struct) - error("Unable to find a struct $onetomany_type_name_w_module") - end + return Dict( + :modules => modules, + :structs => structs, + :fields => fields, + :enums => enums + ) - # Add the fields to the other fields - # NOTE: There is no risk to creaet an infinite loop because we loop over - # the filtered array and not the array itself - push!(fields,onetomany_field) - end +end +function generate_enums_from_object_model(object_model::Dict, outdir::String) - return Dict(:modules => modules, - :structs => structs, - :fields => fields, - :enums => enums) + outdir = joinpath(outdir,"enum") + enums = object_model[:enums] + + result = "" + for e in enums + result *= "module $(e[:module_name])\n" + result *= " export $(e[:type_name])\n" + result *= " @enum $(e[:type_name]) begin\n" + + counter = 0 + for v in e[:values] + counter += 1 + result *= " $v = $counter \n" + end + + result *= " end\n" # close type + result *= "end\n\n" # close module + end + + if !isdir(outdir) + mkpath(outdir) + end + # @info file_path + file_path = joinpath(outdir,"enums.jl") + write(file_path,result) end -function generate_enums_from_object_model(object_model::Dict, outdir::String) +function add_comment_before_of_after(str::String, comment::String) + if length(str) + length(comment) + 3 > 92 + result = insert_newlines_preserving_words(comment, " # ") + result *= "\n" + result *= "$str" + result = "\n$result\n" # Add some additional line returns for better readability + else + result = "$str # $comment" + end + return result +end - outdir = joinpath(outdir,"enum") - enums = object_model[:enums] +function insert_newlines_preserving_words(s::String, prefix::String; n::Int=92) + # Split the string into words + words = split(s) + # Initialize an empty string to build the result result = "" - for e in enums - result *= "module $(e[:module_name])\n" - result *= " export $(e[:type_name])\n" - result *= " @enum $(e[:type_name]) begin\n" - - counter = 0 - for v in e[:values] - counter += 1 - result *= " $v = $counter \n" - end - result *= " end\n" # close type - result *= "end\n\n" # close module + # Initialize a line buffer and a character counter + line = "" + char_count = 0 + + for word in words + # Check if adding the next word exceeds the limit + if char_count + length(word) + (char_count > 0 ? 1 : 0) > n + # Add the current line with prefix to the result and reset it + result *= prefix * line * "\n" + line = word # Start a new line with the current word + char_count = length(word) + else + # Add the word to the current line + line *= (char_count > 0 ? " " : "") * word + char_count += length(word) + (char_count > 0 ? 1 : 0) # +1 for the space if not the first word + end end - if !isdir(outdir) - mkpath(outdir) + # Add the last line with prefix to the result + if !isempty(line) + result *= prefix * line end - # @info file_path - file_path = joinpath(outdir,"enums.jl") - write(file_path,result) + return result end -function generate_structs_from_object_model(object_model::Dict, outdir::String) +function generate_structs_from_object_model( + object_model::Dict, + outdir::String + ;with_comment::Bool = true +) - outdir = joinpath(outdir,"structs") + outdir = joinpath(outdir,"structs") - modules = object_model[:modules] - structs = object_model[:structs] - fields = object_model[:fields] + modules = object_model[:modules] + structs = object_model[:structs] + fields = object_model[:fields] - indent = " " + indent = " " - # Reset content - for _struct in structs - _struct[:struct_content] = "" - end + # Reset content + for _struct in structs + _struct[:struct_content] = "" + end - # ################ # - # Open the struct # - # ################ # - for _struct in structs - str = "mutable struct $(_struct[:name]) <: $(_struct[:abstract_name]) \n\n" - _struct[:struct_content] *= str - end + # ############################################ # + # Add table comment as docstring of the struct # + # ############################################ # + if with_comment + for _struct in structs + if !ismissing(_struct[:comment]) + str = "\"\"\"\n" + str *= "$(_struct[:comment])\n" + str *= "\"\"\"\n" + _struct[:struct_content] *= str + end + end + end - # ################## # - # Declare the fields # - # ################## # - for f in fields + # ################ # + # Open the struct # + # ################ # + for _struct in structs + str = "mutable struct $(_struct[:name]) <: $(_struct[:abstract_name]) \n\n" + _struct[:struct_content] *= str + end - if !haskey(f,:struct) - @error f - return - end + # ################## # + # Declare the fields # + # ################## # + for f in fields - _struct = f[:struct] + if !haskey(f,:struct) + @error f + return + end - field_name = f[:name] - field_type = if (f[:is_manytoone] || f[:is_onetoone] || f[:is_onetomany]) - f[:field_abstract_type] - else - f[:field_type] - end - str = " $field_name::Union{Missing,$field_type}\n" + _struct = f[:struct] - _struct[:struct_content] *= str - end + field_name = f[:name] + field_type = if (f[:is_manytoone] || f[:is_onetoone] || f[:is_onetomany]) + f[:field_abstract_type] + else + f[:field_type] + end + str = " $field_name::Union{Missing,$field_type}" + length_of_field_definition = length(str) + if :comment ∉ keys(f) + @warn "Missing comment key for field[$(f[:name])] $(f[:is_onetomany]), $(f[:is_manytoone]), $(f[:is_onetoone])" + else + if with_comment && !ismissing(f[:comment]) + str = add_comment_before_of_after(str,f[:comment]) + end + end + str *= "\n" + _struct[:struct_content] *= str + end - # ############################ # - # Create the first constructor # - # ############################ # - for _struct in structs - str = "\n" - str *= " $(_struct[:name])(args::NamedTuple) = $(_struct[:name])(;args...)" - _struct[:struct_content] *= str - end + # ############################ # + # Create the first constructor # + # ############################ # + for _struct in structs + str = "\n" + str *= " $(_struct[:name])(args::NamedTuple) = $(_struct[:name])(;args...)" + _struct[:struct_content] *= str + end - # ########################### # - # Open the second constructor # - # ########################### # - for _struct in structs - str = "\n" - str *= " $(_struct[:name])(;\n" - _struct[:struct_content] *= str - end + # ########################### # + # Open the second constructor # + # ########################### # + for _struct in structs + str = "\n" + str *= " $(_struct[:name])(;\n" + _struct[:struct_content] *= str + end - # ####################################################### # - # Add the arguments declaration of the second constructor # - # ####################################################### # - for f in fields + # ####################################################### # + # Add the arguments declaration of the second constructor # + # ####################################################### # + for f in fields - _struct = f[:struct] + _struct = f[:struct] - field_name = f[:name] - str = " $field_name = missing,\n" - _struct[:struct_content] *= str - end + field_name = f[:name] + str = " $field_name = missing,\n" + _struct[:struct_content] *= str + end - # ######################################################### # - # Close the arguments declaration of the second constructor # - # ######################################################### # - for _struct in structs - str = "" - str *= " ) = begin\n" - str *= " x = new(" - _struct[:struct_content] *= str - end + # ######################################################### # + # Close the arguments declaration of the second constructor # + # ######################################################### # + for _struct in structs + str = "" + str *= " ) = begin\n" + str *= " x = new(" + _struct[:struct_content] *= str + end - # #################################################################################### # - # Add the 'new(missing, missing, ...)' arguments assignment of the second constructor # - # #################################################################################### # - for f in fields + # #################################################################################### # + # Add the 'new(missing, missing, ...)' arguments assignment of the second constructor # + # #################################################################################### # + for f in fields - _struct = f[:struct] + _struct = f[:struct] - str = "missing," - _struct[:struct_content] *= str - end + str = "missing," + _struct[:struct_content] *= str + end - # ############################################################ # - # Close 'new(missing, missing, ...)' of the second constructor # - # ############################################################ # - for _struct in structs - str = ")\n" - _struct[:struct_content] *= str - end + # ############################################################ # + # Close 'new(missing, missing, ...)' of the second constructor # + # ############################################################ # + for _struct in structs + str = ")\n" + _struct[:struct_content] *= str + end - # ###################################################### # - # Add the arguments assignment of the second constructor # - # ###################################################### # - for f in fields + # ###################################################### # + # Add the arguments assignment of the second constructor # + # ###################################################### # + for f in fields - _struct = f[:struct] + _struct = f[:struct] - str = " x.$(f[:name]) = $(f[:name])\n" - _struct[:struct_content] *= str - end + str = " x.$(f[:name]) = $(f[:name])\n" + _struct[:struct_content] *= str + end - # ######################################################## # - # Close the arguments assignment of the second constructor # - # ######################################################## # - for _struct in structs - str = " return x\n" - str *= " end\n" - _struct[:struct_content] *= str - end + # ######################################################## # + # Close the arguments assignment of the second constructor # + # ######################################################## # + for _struct in structs + str = " return x\n" + str *= " end\n" + _struct[:struct_content] *= str + end - # ################ # - # Close the struct # - # ################ # - for _struct in structs - str = "\nend " - _struct[:struct_content] *= str - end + # ################ # + # Close the struct # + # ################ # + for _struct in structs + str = "\nend " + _struct[:struct_content] *= str + end - # ############## # - # Write to files # - # ############## # + # ############## # + # Write to files # + # ############## # - # Empty the modules dirs - for _module in modules - module_dir = joinpath(outdir,_module[:name]) - rm(module_dir, recursive=true, force = true) - mkpath(module_dir) - end + # Empty the modules dirs + for _module in modules + module_dir = joinpath(outdir,_module[:name]) + rm(module_dir, recursive=true, force = true) + mkpath(module_dir) + end - # Write abstract types - for _struct in structs - str = "abstract type $(_struct[:abstract_name]) <: IEntity end\n" - module_dir = joinpath(outdir,_struct[:module][:name]) - file_path = joinpath(module_dir,"abstract-types.jl") - io = open(file_path, "a"); - write(io,str) - close(io); - end + # Write abstract types + for _struct in structs + str = "abstract type $(_struct[:abstract_name]) <: IEntity end\n" + module_dir = joinpath(outdir,_struct[:module][:name]) + file_path = joinpath(module_dir,"abstract-types.jl") + io = open(file_path, "a"); + write(io,str) + close(io); + end - # Write the content of structs to files - for _struct in structs - module_dir = joinpath(outdir,_struct[:module][:name]) - if !isdir(module_dir) - mkpath(module_dir) - end - file_path = joinpath(module_dir,"$(_struct[:name]).jl") - write(file_path,_struct[:struct_content]) - end + # Write the content of structs to files + for _struct in structs + module_dir = joinpath(outdir,_struct[:module][:name]) + if !isdir(module_dir) + mkpath(module_dir) + end + file_path = joinpath(module_dir,"$(_struct[:name]).jl") + + # Cleaning of too many line returns (this can happen when adding a field comment) + _struct[:struct_content] = replace(_struct[:struct_content], "\n\n\n\n" => "\n\n") + _struct[:struct_content] = replace(_struct[:struct_content], "\n\n\n" => "\n\n") + + write(file_path,_struct[:struct_content]) + end end function generate_orms_from_object_model(object_model::Dict, outdir::String) - outdir = joinpath(outdir,"orms") + outdir = joinpath(outdir,"orms") - modules = object_model[:modules] - structs = object_model[:structs] - fields = object_model[:fields] + modules = object_model[:modules] + structs = object_model[:structs] + fields = object_model[:fields] - # Check how may different modules we have, if only one, there is no need - # to put the ORMs' modules in separate submodules of the ORM root module - # (i.e. we don't need a module ORM.Public, ORM.SchmeName1, ...) - nb_different_modules = length(unique(x -> x[:name],modules)) + # Check how may different modules we have, if only one, there is no need + # to put the ORMs' modules in separate submodules of the ORM root module + # (i.e. we don't need a module ORM.Public, ORM.SchmeName1, ...) + nb_different_modules = length(unique(x -> x[:name],modules)) - orm_root_module = "ORM" + orm_root_module = "ORM" - indent = " " + indent = " " - # ####################################### # - # Data type, ORM, schema name, table name # - # ####################################### # + # ####################################### # + # Data type, ORM, schema name, table name # + # ####################################### # - for _struct in object_model[:structs] + for _struct in object_model[:structs] - struct_name = "$(_struct[:module][:name]).$(_struct[:name])" - orm_name = if nb_different_modules > 1 - "$(orm_root_module).$(_struct[:module][:name]).$(_struct[:name])ORM" - else - "$(orm_root_module).$(_struct[:name])ORM" - end + struct_name = "$(_struct[:module][:name]).$(_struct[:name])" + orm_name = if nb_different_modules > 1 + "$(orm_root_module).$(_struct[:module][:name]).$(_struct[:name])ORM" + else + "$(orm_root_module).$(_struct[:name])ORM" + end - table = _struct[:table] - schema = _struct[:schema] + table = _struct[:table] + schema = _struct[:schema] - orm_content = (" + orm_content = (" data_type = $struct_name PostgresORM.get_orm(x::$struct_name) = return($(orm_name)) get_schema_name() = \"$schema\" get_table_name() = \"$table\" ") - _struct[:orm_content] = orm_content - _struct[:orm_name] = orm_name + _struct[:orm_content] = orm_content + _struct[:orm_name] = orm_name - # write(filename_for_orm_module,orm_content) + # write(filename_for_orm_module,orm_content) - end #ENDOF `for _struct in structs` + end #ENDOF `for _struct in structs` - # ############################# # - # Columns selection and mapping # - # ############################# # + # ############################# # + # Columns selection and mapping # + # ############################# # - # Declare the function and open the Dict - for _struct in object_model[:structs] - str = "\n\n" - str *= "# Declare the mapping between the properties and the database columns\n" - str *= "get_columns_selection_and_mapping() = return columns_selection_and_mapping" - str *= "\nconst columns_selection_and_mapping = Dict(\n" - _struct[:orm_content] *= str - end #ENDOF `for _struct in structs` + # Declare the function and open the Dict + for _struct in object_model[:structs] + str = "\n\n" + str *= "# Declare the mapping between the properties and the database columns\n" + str *= "get_columns_selection_and_mapping() = return columns_selection_and_mapping" + str *= "\nconst columns_selection_and_mapping = Dict(\n" + _struct[:orm_content] *= str + end #ENDOF `for _struct in structs` - mapping_arr = [] - for f in fields + mapping_arr = [] + for f in fields - # Skip the onetomany fields because they do not have any correson column - if f[:is_onetomany] continue end + # Skip the onetomany fields because they do not have any correson column + if f[:is_onetomany] continue end - field_name = f[:name] - # colnames = "[$(join(f[:cols],", "))]" + field_name = f[:name] + # colnames = "[$(join(f[:cols],", "))]" - colnames = if length(f[:cols]) > 1 - "[" * join( string.("\"", f[:cols], "\""), ", ") * "]" - else - "\"$(f[:cols][1])\"" - end - # colnames = "r4r4W" - str = "$(repeat(indent,1)) :$(field_name) => $colnames, \n" - f[:struct][:orm_content] *= str + colnames = if length(f[:cols]) > 1 + "[" * join( string.("\"", f[:cols], "\""), ", ") * "]" + else + "\"$(f[:cols][1])\"" + end + # colnames = "r4r4W" + str = "$(repeat(indent,1)) :$(field_name) => $colnames, \n" + f[:struct][:orm_content] *= str - end # ENDOF `for id_field in object_model[:id_fields]` + end # ENDOF `for id_field in object_model[:id_fields]` - # Close the Dict for 'columns_selection_and_mapping' - for _struct in object_model[:structs] - _struct[:orm_content] *= ")\n\n" - end + # Close the Dict for 'columns_selection_and_mapping' + for _struct in object_model[:structs] + _struct[:orm_content] *= ")\n\n" + end - # ############# # - # ID properties # - # ############# # + # ############# # + # ID properties # + # ############# # - # Open the function - for _struct in object_model[:structs] - _struct[:orm_content] *= "\n" - _struct[:orm_content] *= "# Declare which properties are used to uniquely identify an object\n" - _struct[:orm_content] *= "get_id_props() = return [" - end + # Open the function + for _struct in object_model[:structs] + _struct[:orm_content] *= "\n" + _struct[:orm_content] *= "# Declare which properties are used to uniquely identify an object\n" + _struct[:orm_content] *= "get_id_props() = return [" + end - # Add the fieds - for f in filter(x -> x[:is_id], fields) - f[:struct][:orm_content] *= ":$(f[:name])," - end + # Add the fieds + for f in filter(x -> x[:is_id], fields) + f[:struct][:orm_content] *= ":$(f[:name])," + end - # Close the function - for _struct in object_model[:structs] - _struct[:orm_content] *= "]" - end + # Close the function + for _struct in object_model[:structs] + _struct[:orm_content] *= "]" + end - # ###################### # - # onetomany_counterparts # - # ###################### # - # Declare the function and open the Dict - for _struct in object_model[:structs] - _struct[:orm_content] *= "\n\n" - _struct[:orm_content] *= "# Associate the onetomany properties to the corresponding manytoone peroperties in the other classes \n" - _struct[:orm_content] *= "get_onetomany_counterparts() = return onetomany_counterparts\n" - _struct[:orm_content] *= "const onetomany_counterparts = Dict(\n" - end + # ###################### # + # onetomany_counterparts # + # ###################### # + # Declare the function and open the Dict + for _struct in object_model[:structs] + _struct[:orm_content] *= "\n\n" + _struct[:orm_content] *= "# Associate the onetomany properties to the corresponding manytoone peroperties in the other classes \n" + _struct[:orm_content] *= "get_onetomany_counterparts() = return onetomany_counterparts\n" + _struct[:orm_content] *= "const onetomany_counterparts = Dict(\n" + end - # Add the fieds - for f in filter(x -> x[:is_onetomany], fields) + # Add the fieds + for f in filter(x -> x[:is_onetomany], fields) - field_name = f[:name] - field_type = f[:field_type] - manytoone = f[:manytoone_field] - manytoone_struct = - "$(manytoone[:struct][:module][:name]).$(manytoone[:struct][:name])" - manytoone_field_name = manytoone[:name] + field_name = f[:name] + field_type = f[:field_type] + manytoone = f[:manytoone_field] + manytoone_struct = + "$(manytoone[:struct][:module][:name]).$(manytoone[:struct][:name])" + manytoone_field_name = manytoone[:name] - tip_for_data_type = "# The struct where the associated manytoone property is" - tip_for_property = "# The name of the associated manytoone property" - tip_for_action_on_remove = - "# Change this to 'PostgresORM.CRUDType.delete' if the object doesn't make sense when orphaned" + tip_for_data_type = "# The struct where the associated manytoone property is" + tip_for_property = "# The name of the associated manytoone property" + tip_for_action_on_remove = + "# Change this to 'PostgresORM.CRUDType.delete' if the object doesn't make sense when orphaned" - str = " + str = " :$(field_name) => ( data_type = $manytoone_struct, $tip_for_data_type property = :$manytoone_field_name, $tip_for_property action_on_remove = PostgresORM.CRUDType.update), $tip_for_action_on_remove \n" f[:struct][:orm_content] *= str - end + end - # Close the Dict - for _struct in object_model[:structs] - _struct[:orm_content] *= "\n)" - end + # Close the Dict + for _struct in object_model[:structs] + _struct[:orm_content] *= "\n)" + end - # ############## # - # Types override # - # ############## # - # Declare the function and open the Dict - for _struct in object_model[:structs] - _struct[:orm_content] *= "\n\n" - _struct[:orm_content] *= "# Override the abstract types \n" - _struct[:orm_content] *= "get_types_override() = return types_override\n" - _struct[:orm_content] *= "const types_override = Dict(\n" - end + # ############## # + # Types override # + # ############## # + # Declare the function and open the Dict + for _struct in object_model[:structs] + _struct[:orm_content] *= "\n\n" + _struct[:orm_content] *= "# Override the abstract types \n" + _struct[:orm_content] *= "get_types_override() = return types_override\n" + _struct[:orm_content] *= "const types_override = Dict(\n" + end - # Add the fieds for the manytoone and onetomany fields - for f in filter(x -> (x[:is_manytoone] || x[:is_onetoone] || x[:is_onetomany]), - fields) - field_name = f[:name] - field_type = f[:field_type] - str = "$(repeat(indent,1)) :$(field_name) => $field_type, \n" - f[:struct][:orm_content] *= str - end + # Add the fieds for the manytoone and onetomany fields + for f in filter(x -> (x[:is_manytoone] || x[:is_onetoone] || x[:is_onetomany]), + fields) + field_name = f[:name] + field_type = f[:field_type] + str = "$(repeat(indent,1)) :$(field_name) => $field_type, \n" + f[:struct][:orm_content] *= str + end - # Close the Dict - for _struct in object_model[:structs] - _struct[:orm_content] *= "\n)" - end + # Close the Dict + for _struct in object_model[:structs] + _struct[:orm_content] *= "\n)" + end - # ############# # - # Track changes # - # ############# # - # Declare the function and open the Dict - for _struct in object_model[:structs] - _struct[:orm_content] *= "\n\n" - _struct[:orm_content] *= "# Specify whether we want to track the changes to the objects of this class \n" - _struct[:orm_content] *= "# get_track_changes() = false # Uncomment and modify if needed \n" - _struct[:orm_content] *= "# get_creator_property() = :a_property_symbol # Uncomment and modify if needed \n" - _struct[:orm_content] *= "# get_editor_property() = :a_property_symbol # Uncomment and modify if needed \n" - _struct[:orm_content] *= "# get_creation_time_property() = :a_property_symbol # Uncomment and modify if needed \n" - _struct[:orm_content] *= "# get_update_time_property() = :a_property_symbol # Uncomment and modify if needed \n" - end + # ############# # + # Track changes # + # ############# # + # Declare the function and open the Dict + for _struct in object_model[:structs] + _struct[:orm_content] *= "\n\n" + _struct[:orm_content] *= "# Specify whether we want to track the changes to the objects of this class \n" + _struct[:orm_content] *= "# get_track_changes() = false # Uncomment and modify if needed \n" + _struct[:orm_content] *= "# get_creator_property() = :a_property_symbol # Uncomment and modify if needed \n" + _struct[:orm_content] *= "# get_editor_property() = :a_property_symbol # Uncomment and modify if needed \n" + _struct[:orm_content] *= "# get_creation_time_property() = :a_property_symbol # Uncomment and modify if needed \n" + _struct[:orm_content] *= "# get_update_time_property() = :a_property_symbol # Uncomment and modify if needed \n" + end - # ############## # - # Write to files # - # ############## # + # ############## # + # Write to files # + # ############## # - # Empty the modules dirs - for _module in modules - module_dir = joinpath(outdir,_module[:name]) - rm(module_dir, recursive=true, force = true) - mkpath(module_dir) - end + # Empty the modules dirs + for _module in modules + module_dir = joinpath(outdir,_module[:name]) + rm(module_dir, recursive=true, force = true) + mkpath(module_dir) + end - # Write abstract types - for _struct in structs - module_dir = joinpath(outdir,_struct[:module][:name]) - orm_name_wo_module = - _struct[:orm_name][ - findlast(".",_struct[:orm_name])[1]+1:length(_struct[:orm_name]) - ] - file_path = joinpath(module_dir,"$orm_name_wo_module.jl") - io = open(file_path, "w"); - write(io,_struct[:orm_content]) - close(io); - end + # Write abstract types + for _struct in structs + module_dir = joinpath(outdir,_struct[:module][:name]) + orm_name_wo_module = + _struct[:orm_name][ + findlast(".",_struct[:orm_name])[1]+1:length(_struct[:orm_name]) + ] + file_path = joinpath(module_dir,"$orm_name_wo_module.jl") + io = open(file_path, "w"); + write(io,_struct[:orm_content]) + close(io); + end - return object_model + return object_model end From a58051753696fd6d89607f9561d4d80f25dd6f71 Mon Sep 17 00:00:00 2001 From: Vincent Laugier Date: Sat, 16 Mar 2024 15:22:26 +0000 Subject: [PATCH 2/7] 'one_to_one' in the FK name is also interpreted as a 'one-to-one' relation --- src/Tool/Tool.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tool/Tool.jl b/src/Tool/Tool.jl index 22b8847..8cb75b5 100644 --- a/src/Tool/Tool.jl +++ b/src/Tool/Tool.jl @@ -255,7 +255,7 @@ function generate_object_model( manytoone_field[:referenced_cols] = fkdef[:referenced_cols] manytoone_field[:cols] = fkdef[:referencing_cols] manytoone_field[:is_id] = false - if occursin("onetoone",fkname) + if occursin("onetoone",fkname) || occursin("one_to_one",fkname) manytoone_field[:is_onetoone] = true manytoone_field[:is_manytoone] = false else From 63ea1d8c1c751b0a0a43bc77ecb0dc4d2de9866b Mon Sep 17 00:00:00 2001 From: Vincent Laugier Date: Sat, 16 Mar 2024 15:22:56 +0000 Subject: [PATCH 3/7] add utility function dedup_colnames_colvalues! --- src/PostgresORMUtil/utils.jl | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/PostgresORMUtil/utils.jl b/src/PostgresORMUtil/utils.jl index fca58ba..0882b90 100644 --- a/src/PostgresORMUtil/utils.jl +++ b/src/PostgresORMUtil/utils.jl @@ -380,3 +380,59 @@ function remove_spaces_and_split(str::String) str = replace(str, " " => "") return split(str,',') end + +""" + dedup_colnames_colvalues!(column_names::Vector{String}, column_values::Vector) + +Inplace dedup column names and values by giving priority to non missing elements + +Eg. +Applying function to the following arguments: +column_names = ["exam_log_id", "in_date", "in_date", "out_time", "api_url", "in_time", "user_id"] +column_values = ["215c81e9-e002-402e-8482-1382a65ef1e4", "2024-03-11", missing, "2024-03-11T08:27:12.363+01:00", "exam/save-values", "2024-03-11T08:27:09.363+01:00", "b7845f35-0169-488d-b8ce-111f0f07d695"] + +will result in a dropping the third element of both vectors + +""" +function dedup_colnames_colvalues!( + column_names::Vector{String}, + column_values::Vector +) + + # Function to check if a value is considered "missing" in your context + is_missing(value) = value === missing # Replace 'nothing' with your definition of a missing value + + # Create a dictionary to track the first occurrence and its index + seen = Dict{String, Int}() + + # Iterate backwards to prefer the first non-missing value when deduplicating + for i in length(column_names):-1:1 + name, value = column_names[i], column_values[i] + if !is_missing(value) + if haskey(seen, name) + # Update with non-missing value if found later + column_values[seen[name]] = value + else + seen[name] = i # Track the first non-missing occurrence + end + elseif !haskey(seen, name) + # Even if it's missing, we need to track the first occurrence + seen[name] = i + end + end + + # Filter the original arrays based on the indices stored in 'seen' + # This will also sort them based on their original order preserved by iteration order in 'seen' + filter_indices = sort(collect(values(seen))) + + # Assign new values to the beginning of the original arrays + column_names[1:length(filter_indices)] = [column_names[i] for i in filter_indices] + column_values[1:length(filter_indices)] = [column_values[i] for i in filter_indices] + + # Resize the original arrays to remove the extra elements + resize!(column_names, length(filter_indices)) + resize!(column_values, length(filter_indices)) + + nothing + +end From 2481457632f73ec943b4dfa9bce55be7cdf3b414 Mon Sep 17 00:00:00 2001 From: Vincent Laugier Date: Sat, 16 Mar 2024 15:23:42 +0000 Subject: [PATCH 4/7] use dedup_colnames_colvalues! to handle columns that are referenced by both FK and PK --- src/Controller/coreORM.create.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Controller/coreORM.create.jl b/src/Controller/coreORM.create.jl index f4c0353..c60ef5e 100644 --- a/src/Controller/coreORM.create.jl +++ b/src/Controller/coreORM.create.jl @@ -95,6 +95,9 @@ function create_entity!(new_object::IEntity, column_names = util_getcolumns(properties_names, columns_selection_and_mapping) properties_values = collect(values(props)) + # Dedup column names and values in case some properties are sharing the same columns + PostgresORMUtil.dedup_colnames_colvalues!(column_names, properties_values) + # Loop over the properties of the object and # build the appropriate list of columns if length(props) > 0 @@ -104,7 +107,7 @@ function create_entity!(new_object::IEntity, end # Add the prepared statement indexes - query_indexes = string.(collect(1:length(properties_names))) + query_indexes = string.(collect(1:length(column_names))) query_indexes = string.("\$",query_indexes) query_string *= (" VALUES (" * join(query_indexes,",") From f32dc452af1c5db82172ffd3d83aa8a6cf438427 Mon Sep 17 00:00:00 2001 From: Vincent Laugier Date: Sat, 16 Mar 2024 15:27:26 +0000 Subject: [PATCH 5/7] bump from 0.5.7 to 0.6.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index ba5ab7d..57cc2df 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PostgresORM" uuid = "748b5efa-ed57-4836-b183-a38105a77fdd" authors = ["Vincent Laugier "] -version = "0.5.7" +version = "0.6.0" [deps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" From fd80ca36bf81fdee1bf3c9852b89c4876cc6312e Mon Sep 17 00:00:00 2001 From: Vincent Laugier Date: Fri, 19 Apr 2024 17:25:43 +0000 Subject: [PATCH 6/7] fix error message --- src/Controller/coreORM.update.jl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Controller/coreORM.update.jl b/src/Controller/coreORM.update.jl index 8d13c13..95a1acd 100644 --- a/src/Controller/coreORM.update.jl +++ b/src/Controller/coreORM.update.jl @@ -310,10 +310,17 @@ function update_entity!(updated_object::IEntity, dbconn) if length(previous_state) == 0 - throw(DomainError("Unable to retrieve an object of type[$data_type] " - * "with id[$(getproperty(updated_object, - orm_module.id_property))] from the database." - *" Remind that only existing existing objects can be updated.")) + + id_for_display = util_get_ids_cols_names_and_values(updated_object, dbconn) |> + dict -> join(["$k: $v" for (k, v) in dict], ", ") + + throw( + DomainError( + "Unable to retrieve an object of type[$data_type] " + *"with id[$id_for_display] " + *"from the database. Remind that only existing objects can be updated." + ) + ) end previous_state = previous_state[1] From a4cbf1fc6a592676feb4ba3a9deee9470fc25e57 Mon Sep 17 00:00:00 2001 From: Vincent Laugier Date: Fri, 19 Apr 2024 17:26:01 +0000 Subject: [PATCH 7/7] bump --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 57cc2df..426433e 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PostgresORM" uuid = "748b5efa-ed57-4836-b183-a38105a77fdd" authors = ["Vincent Laugier "] -version = "0.6.0" +version = "0.6.1" [deps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"