Skip to content

Commit

Permalink
feat: jans-linux-setup postgresql support (#2409)
Browse files Browse the repository at this point in the history 
* feat: jans-linux-setup postgresql support

* feat: add PostgreSQL support

Co-authored-by: Yuriy Movchan <Yuriy.Movchan@gmail.com>
  • Loading branch information
@devrimyatar @yurem
devrimyatar and yurem committed Sep 16, 2022
1 parent bb334a9 commit 08ecaf9
Show file tree
Hide file tree
Showing 27 changed files with 1,071 additions and 367 deletions.
18 changes: 10 additions & 8 deletions jans-linux-setup/jans_setup/setup_app/installers/rdbm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,10 @@ def __init__(self):
self.install_type = InstallOption.OPTONAL
self.install_var = 'rdbm_install'
self.register_progess()
self.qchar = '`' if Config.rdbm_type in ('mysql', 'spanner') else '"'
self.output_dir = os.path.join(Config.output_dir, Config.rdbm_type)

def install(self):

self.qchar = '`' if Config.rdbm_type in ('mysql', 'spanner') else '"'
self.local_install()
jans_schema_files = []
self.jans_attributes = []
Expand Down Expand Up @@ -179,7 +178,7 @@ def create_tables(self, jans_schema_files):
doc_id_type = self.get_sql_col_type('doc_id', sql_tbl_name)
if Config.rdbm_type == 'pgsql':
sql_cmd = 'CREATE TABLE "{}" (doc_id {} NOT NULL UNIQUE, "objectClass" VARCHAR(48), dn VARCHAR(128), {}, PRIMARY KEY (doc_id));'.format(sql_tbl_name, doc_id_type, ', '.join(sql_tbl_cols))
if Config.rdbm_type == 'spanner':
elif Config.rdbm_type == 'spanner':
sql_cmd = 'CREATE TABLE `{}` (`doc_id` {} NOT NULL, `objectClass` STRING(48), dn STRING(128), {}) PRIMARY KEY (`doc_id`)'.format(sql_tbl_name, doc_id_type, ', '.join(sql_tbl_cols))
else:
sql_cmd = 'CREATE TABLE `{}` (`doc_id` {} NOT NULL UNIQUE, `objectClass` VARCHAR(48), dn VARCHAR(128), {}, PRIMARY KEY (`doc_id`));'.format(sql_tbl_name, doc_id_type, ', '.join(sql_tbl_cols))
Expand Down Expand Up @@ -235,7 +234,7 @@ def create_indexes(self):
if Config.rdbm_type == 'spanner':
tables = self.dbUtils.spanner.get_tables()
for tblCls in tables:
tbl_fields = sql_indexes.get(tblCls, {}).get('fields', []) + sql_indexes['__common__']['fields']
tbl_fields = sql_indexes.get(tblCls, {}).get('fields', []) + sql_indexes['__common__']['fields']

tbl_data = self.dbUtils.spanner.exec_sql('SELECT * FROM {} LIMIT 1'.format(tblCls))

Expand Down Expand Up @@ -293,7 +292,7 @@ def create_indexes(self):
)
self.dbUtils.exec_rdbm_query(sql_cmd)
elif Config.rdbm_type == 'pgsql':
sql_cmd ='CREATE INDEX ON "{}" (({}));'.format(
sql_cmd ='CREATE INDEX ON "{}" {};'.format(
tblCls,
tmp_str.safe_substitute({'field':attr.name})
)
Expand Down Expand Up @@ -326,7 +325,7 @@ def create_indexes(self):
)
self.dbUtils.exec_rdbm_query(sql_cmd)
elif Config.rdbm_type == 'pgsql':
sql_cmd = 'CREATE INDEX ON "{}" ("{}");'.format(
sql_cmd = 'CREATE INDEX ON "{}" {};'.format(
tblCls,
custom_index
)
Expand Down Expand Up @@ -362,9 +361,12 @@ def import_ldif(self):
self.dbUtils.import_ldif(ldif_files)

def rdbmProperties(self):
if Config.rdbm_type in ('sql', 'mysql'):
if Config.rdbm_type in ('pgsql', 'mysql'):
Config.rdbm_password_enc = self.obscure(Config.rdbm_password)
self.renderTemplateInOut(Config.jansRDBMProperties, Config.templateFolder, Config.configFolder)
src_temp_fn = os.path.join(Config.templateFolder, 'jans-{}.properties'.format(Config.rdbm_type))
targtet_fn = os.path.join(Config.configFolder, Config.jansRDBMProperties)
rendered_tmp = self.render_template(src_temp_fn)
self.writeFile(targtet_fn, rendered_tmp)

elif Config.rdbm_type == 'spanner':
if Config.spanner_emulator_host:
Expand Down
12 changes: 12 additions & 0 deletions jans-linux-setup/jans_setup/setup_app/static.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,18 @@ class BackendTypes:
PGSQL = 4
SPANNER = 5

class BackendStrings:
LOCAL_OPENDJ = 'Local OpenDj'
REMOTE_OPENDJ = 'Remote OpenDj'
LOCAL_COUCHBASE = 'Local Couchbase'
REMOTE_COUCHBASE = 'Remote Couchbase'
LOCAL_MYSQL = 'Local MySQL'
REMOTE_MYSQL = 'Remote MySQL'
CLOUD_SPANNER = 'Cloud Spanner'
SAPNNER_EMULATOR = 'Spanner Emulator'
LOCAL_PGSQL = 'Local PgSQL'
REMOTE_PGSQL = 'Remote PgSQL'

class AppType:
APPLICATION = 1
SERVICE = 2
Expand Down
2 changes: 2 additions & 0 deletions jans-linux-setup/jans_setup/setup_app/utils/collect_properties.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ def collect(self):
Config.rdbm_password_enc = jans_sql_prop['auth.userPassword']
Config.rdbm_password = self.unobscure(Config.rdbm_password_enc)
Config.rdbm_db = jans_sql_prop['db.schema.name']
if Config.rdbm_type == 'postgresql':
Config.rdbm_type = 'pgsql'

if not Config.persistence_type in ('couchbase', 'ldap') and Config.get('jansSpannerProperties') and os.path.exists(Config.jansSpannerProperties):
Config.rdbm_type = 'spanner'
Expand Down
41 changes: 26 additions & 15 deletions jans-linux-setup/jans_setup/setup_app/utils/db_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def bind(self, use_ssl=True, force=False):
for group in Config.mapping_locations:
if Config.mapping_locations[group] == 'rdbm':
if Config.rdbm_type in ('mysql', 'pgsql'):
base.logIt("Making MySql Conncetion")
base.logIt("Making {} Conncetion".format(Config.rdbm_type))
result = self.mysqlconnection()
if not result[0]:
print("{}FATAL: {}{}".format(colors.FAIL, result[1], colors.ENDC))
Expand Down Expand Up @@ -131,7 +131,7 @@ def sqlconnection(self, log=True):

@property
def json_dialects_instance(self):
return sqlalchemy.dialects.mysql.json.JSON if Config.rdbm_type == 'mysql' else sqlalchemy.dialects.postgresql.json.JSON
return sqlalchemy.dialects.mysql.json.JSON if Config.rdbm_type == 'mysql' else sqlalchemy.dialects.postgresql.json.JSONB

def mysqlconnection(self, log=True):
return self.sqlconnection(log)
Expand All @@ -151,7 +151,7 @@ def read_jans_schema(self, others=[]):

for attr in attribDataTypes.listAttributes:
if not attr in self.sql_data_types:
self.sql_data_types[attr] = { 'mysql': {'type': 'JSON'}, 'spanner': {'type': 'ARRAY<STRING(MAX)>'} }
self.sql_data_types[attr] = { 'mysql': {'type': 'JSON'}, 'pgsql': {'type': 'JSONB'}, 'spanner': {'type': 'ARRAY<STRING(MAX)>'} }

def in_subtable(self, table, attr):
if table in self.sub_tables[Config.rdbm_type]:
Expand Down Expand Up @@ -355,7 +355,7 @@ def dn_exists_rdbm(self, dn, table):
return result
return
sqlalchemy_table = self.Base.classes[table].__table__
return self.session.query(sqlalchemy_table).filter(sqlalchemy_table).filter(sqlalchemy_table.columns.dn == dn).first()
return self.session.query(sqlalchemy_table).filter(sqlalchemy_table.columns.dn == dn).first()


def spanner_to_dict(self, data):
Expand Down Expand Up @@ -577,14 +577,19 @@ def add_client2script(self, script_inum, client_id):
else:
jansConfProperty = {'v': []}

for oxconfigprop in jansConfProperty['v']:
if oxconfigprop.get('value1') == 'allowed_clients' and not client_id in oxconfigprop['value2']:
ox_configuration_property_list = jansConfProperty['v'] if Config.rdbm_type == 'mysql' else jansConfProperty

for i, oxconfigprop in enumerate(ox_configuration_property_list[:]):
if isinstance(oxconfigprop, str):
oxconfigprop = json.loads(oxconfigprop)
if oxconfigprop.get('value1') == 'allowed_clients' and client_id not in oxconfigprop['value2']:
oxconfigprop['value2'] = self.add2strlist(client_id, oxconfigprop['value2'])
ox_configuration_property_list[i] = json.dumps(oxconfigprop)
break
else:
jansConfProperty['v'].append({'value1': 'allowed_clients', 'value2': client_id})
ox_configuration_property_list.append(json.dumps({'value1': 'allowed_clients', 'value2': client_id}))

sqlalchemyObj.jansConfProperty = jansConfProperty
sqlalchemyObj.jansConfProperty = jansConfProperty if BackendTypes.MYSQL else ox_configuration_property_list
self.session.commit()


Expand Down Expand Up @@ -683,11 +688,13 @@ def rdm_automapper(self, force=False):
self.Base = sqlalchemy.ext.automap.automap_base(metadata=self.metadata)
self.Base.prepare()


# fix JSON type for mariadb
for tbl in self.Base.classes:
for col in tbl.__table__.columns:
if isinstance(col.type, sqlalchemy.dialects.mysql.LONGTEXT) and col.comment.lower() == 'json':
col.type = sqlalchemy.dialects.mysql.json.JSON()
if Config.rdbm_type == 'mysql':
for tbl in self.Base.classes:
for col in tbl.__table__.columns:
if isinstance(col.type, sqlalchemy.dialects.mysql.LONGTEXT) and col.comment.lower() == 'json':
col.type = sqlalchemy.dialects.mysql.json.JSON()

base.logIt("Reflected tables {}".format(list(self.metadata.tables.keys())))

Expand Down Expand Up @@ -730,7 +737,7 @@ def get_rdbm_val(self, key, val, rdbm_type=None):

data_type = self.get_attr_sql_data_type(key)

if data_type in ('SMALLINT', 'BOOL'):
if data_type in ('SMALLINT', 'BOOL', 'BOOLEAN'):
if val[0].lower() in ('1', 'on', 'true', 'yes', 'ok'):
return 1 if data_type == 'SMALLINT' else True
return 0 if data_type == 'SMALLINT' else False
Expand All @@ -751,7 +758,7 @@ def get_rdbm_val(self, key, val, rdbm_type=None):

return json_data

if data_type == 'ARRAY<STRING(MAX)>':
if data_type in ('ARRAY<STRING(MAX)>', 'JSONB'):
return val

return val[0]
Expand Down Expand Up @@ -835,6 +842,10 @@ def import_ldif(self, ldif_files, bucket=None, force=None):
cur_val = copy.deepcopy(getattr(sqlalchObj, attribute))
for val_ in new_val:
cur_val['v'].append(val_)
if Config.rdbm_type == 'mysql':
cur_val['v'].append(val_)
else:
cur_val.append(val_)
setattr(sqlalchObj, attribute, cur_val)
else:
setattr(sqlalchObj, attribute, new_val[0])
Expand Down Expand Up @@ -888,7 +899,7 @@ def import_ldif(self, ldif_files, bucket=None, force=None):

for col in sqlalchCls.__table__.columns:
if isinstance(col.type, self.json_dialects_instance) and not col.name in vals:
vals[col.name] = {'v': []}
vals[col.name] = {'v': []} if Config.rdbm_type == 'mysql' else []

sqlalchObj = sqlalchCls()

Expand Down
77 changes: 46 additions & 31 deletions jans-linux-setup/jans_setup/setup_app/utils/properties_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,15 @@
import ssl
import re
import pymysql
import psycopg2
import inspect
import ldap3

from setup_app import paths
from setup_app.messages import msg
from setup_app.utils import base
from setup_app.utils.cbm import CBM
from setup_app.static import InstallTypes, colors
from setup_app.static import InstallTypes, colors, BackendStrings

from setup_app.config import Config
from setup_app.utils.setup_utils import SetupUtils
Expand Down Expand Up @@ -659,16 +660,18 @@ def prompt_for_rdbm(self):
def prompt_for_backend(self):
print('Chose Backend Type:')

backend_types = ['Local OpenDj',
'Remote OpenDj',
'Local MySQL',
'Remote MySQL',
]
backend_types = [
BackendStrings.LOCAL_OPENDJ,
BackendStrings.LOCAL_MYSQL,
BackendStrings.REMOTE_MYSQL,
BackendStrings.LOCAL_PGSQL,
BackendStrings.REMOTE_PGSQL,
]

if not os.path.exists(os.path.join(Config.install_dir, 'package')):
backend_types += ['Remote Couchbase', 'Cloud Spanner']
backend_types += [BackendStrings.REMOTE_COUCHBASE, BackendStrings.CLOUD_SPANNER]
if 'couchbase' in self.getBackendTypes():
backend_types.insert(2, 'Local Couchbase')
backend_types.insert(2, BackendStrings.LOCAL_COUCHBASE)

nlist = []
for i, btype in enumerate(backend_types):
Expand Down Expand Up @@ -697,7 +700,7 @@ def prompt_for_backend(self):

backend_type_str = backend_types[int(choice)-1]

if backend_type_str == 'Local OpenDj':
if backend_type_str == BackendStrings.LOCAL_OPENDJ:
Config.opendj_install = InstallTypes.LOCAL
ldapPass = Config.ldapPass or Config.admin_password or self.getPW(special='.*=!%&+/-')

Expand All @@ -712,7 +715,7 @@ def prompt_for_backend(self):
Config.ldapPass = ldapPass


elif backend_type_str == 'Remote OpenDj':
elif backend_type_str == BackendStrings.REMOTE_OPENDJ:
Config.opendj_install = InstallTypes.REMOTE
while True:
ldapHost = self.getPrompt(" LDAP hostname")
Expand All @@ -726,7 +729,7 @@ def prompt_for_backend(self):
Config.ldapPass = ldapPass
Config.ldap_hostname = ldapHost

elif backend_type_str == 'Local Couchbase':
elif backend_type_str == BackendStrings.LOCAL_COUCHBASE:
Config.opendj_install = InstallTypes.NONE
Config.cb_install = InstallTypes.LOCAL
Config.isCouchbaseUserAdmin = True
Expand All @@ -742,7 +745,7 @@ def prompt_for_backend(self):
Config.cb_password = cbPass
Config.mapping_locations = { group: 'couchbase' for group in Config.couchbaseBucketDict }

elif backend_type_str == 'Remote Couchbase':
elif backend_type_str == BackendStrings.REMOTE_COUCHBASE:
Config.opendj_install = InstallTypes.NONE
Config.cb_install = InstallTypes.REMOTE

Expand All @@ -756,38 +759,50 @@ def prompt_for_backend(self):

Config.mapping_locations = { group: 'couchbase' for group in Config.couchbaseBucketDict }

elif backend_type_str == 'Local MySQL':
elif backend_type_str in (BackendStrings.LOCAL_MYSQL, BackendStrings.LOCAL_PGSQL):
Config.opendj_install = InstallTypes.NONE
Config.rdbm_install = True
Config.rdbm_install_type = InstallTypes.LOCAL
Config.rdbm_type = 'mysql'
Config.rdbm_host = 'localhost'
Config.rdbm_user = 'jans'
Config.rdbm_password = self.getPW(special='.*=+-()[]{}')
Config.rdbm_port = 3306
Config.rdbm_db = 'jansdb'

elif backend_type_str == 'Remote MySQL':
if backend_type_str == BackendStrings.LOCAL_MYSQL:
Config.rdbm_port = 3306
Config.rdbm_type = 'mysql'
else:
Config.rdbm_port = 5432
Config.rdbm_type = 'pgsql'
if not Config.rdbm_host:

if not Config.rdbm_password:
Config.rdbm_password = self.getPW(special='.*=+-()[]{}')

elif backend_type_str in (BackendStrings.REMOTE_MYSQL, BackendStrings.REMOTE_PGSQL):
Config.opendj_install = InstallTypes.NONE
Config.rdbm_install = True
Config.rdbm_install_type = InstallTypes.REMOTE
Config.rdbm_type = 'mysql'
if backend_type_str == BackendStrings.REMOTE_MYSQL:
Config.rdbm_port = 3306
Config.rdbm_type = 'mysql'
else:
Config.rdbm_port = 5432
Config.rdbm_type = 'pgsql'

while True:
Config.rdbm_host = self.getPrompt(" Mysql host", Config.get('rdbm_host'))
Config.rdbm_port = self.getPrompt(" Mysql port", 3306, itype=int, indent=1)
Config.rdbm_user = self.getPrompt(" Mysql user", Config.get('rdbm_user'))
Config.rdbm_password = self.getPrompt(" Mysql password")
Config.rdbm_db = self.getPrompt(" Mysql database", Config.get('rdbm_db'))
Config.rdbm_host = self.getPrompt(" {} host".format(Config.rdbm_type.upper()), Config.get('rdbm_host'))
Config.rdbm_port = self.getPrompt(" {} port".format(Config.rdbm_type.upper()), Config.rdbm_port, itype=int, indent=1)
Config.rdbm_user = self.getPrompt(" {} user".format(Config.rdbm_type.upper()), Config.get('rdbm_user'))
Config.rdbm_password = self.getPrompt(" {} password".format(Config.rdbm_type.upper()))
Config.rdbm_db = self.getPrompt(" {} database".format(Config.rdbm_type.upper()), Config.get('rdbm_db'))

try:
pymysql.connect(host=Config.rdbm_host, user=Config.rdbm_user, password=Config.rdbm_password, database=Config.rdbm_db, port=Config.rdbm_port)
print(" {}MySQL connection was successful{}".format(colors.OKGREEN, colors.ENDC))
if Config.rdbm_type == 'mysql':
pymysql.connect(host=Config.rdbm_host, user=Config.rdbm_user, password=Config.rdbm_password, database=Config.rdbm_db, port=Config.rdbm_port)
else:
psycopg2.connect(dbname=Config.rdbm_db, user=Config.rdbm_user, password=Config.rdbm_password, host=Config.rdbm_host, port=Config.rdbm_port)
print(" {}{} connection was successful{}".format(colors.OKGREEN, Config.rdbm_type.upper(), colors.ENDC))
break
except Exception as e:
print(" {}Can't connect to MySQL: {}{}".format(colors.DANGER, e, colors.ENDC))
print(" {}Can't connect to {}: {}{}".format(colors.DANGER,Config.rdbm_type.upper(), e, colors.ENDC))

elif backend_type_str == 'Cloud Spanner':
elif backend_type_str == BackendStrings.CLOUD_SPANNER:
Config.opendj_install = InstallTypes.NONE
Config.rdbm_type = 'spanner'
Config.rdbm_install = True
Expand Down
Loading

0 comments on commit 08ecaf9

Please sign in to comment.