Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: jans-linux-setup postgresql support #2409

Merged
merged 2 commits into from
Sep 16, 2022
Merged

feat: jans-linux-setup postgresql support #2409

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
13c364c
feat: jans-linux-setup postgresql support
devrimyatar Sep 16, 2022
c59c4cd
feat: add PostgreSQL support
yurem Sep 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions jans-linux-setup/jans_setup/setup_app/installers/rdbm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,10 @@ def __init__(self):
self.install_type = InstallOption.OPTONAL
self.install_var = 'rdbm_install'
self.register_progess()
self.qchar = '`' if Config.rdbm_type in ('mysql', 'spanner') else '"'
self.output_dir = os.path.join(Config.output_dir, Config.rdbm_type)

def install(self):

self.qchar = '`' if Config.rdbm_type in ('mysql', 'spanner') else '"'
self.local_install()
jans_schema_files = []
self.jans_attributes = []
Expand Down Expand Up @@ -179,7 +178,7 @@ def create_tables(self, jans_schema_files):
doc_id_type = self.get_sql_col_type('doc_id', sql_tbl_name)
if Config.rdbm_type == 'pgsql':
sql_cmd = 'CREATE TABLE "{}" (doc_id {} NOT NULL UNIQUE, "objectClass" VARCHAR(48), dn VARCHAR(128), {}, PRIMARY KEY (doc_id));'.format(sql_tbl_name, doc_id_type, ', '.join(sql_tbl_cols))
if Config.rdbm_type == 'spanner':
elif Config.rdbm_type == 'spanner':
sql_cmd = 'CREATE TABLE `{}` (`doc_id` {} NOT NULL, `objectClass` STRING(48), dn STRING(128), {}) PRIMARY KEY (`doc_id`)'.format(sql_tbl_name, doc_id_type, ', '.join(sql_tbl_cols))
else:
sql_cmd = 'CREATE TABLE `{}` (`doc_id` {} NOT NULL UNIQUE, `objectClass` VARCHAR(48), dn VARCHAR(128), {}, PRIMARY KEY (`doc_id`));'.format(sql_tbl_name, doc_id_type, ', '.join(sql_tbl_cols))
Expand Down Expand Up @@ -235,7 +234,7 @@ def create_indexes(self):
if Config.rdbm_type == 'spanner':
tables = self.dbUtils.spanner.get_tables()
for tblCls in tables:
tbl_fields = sql_indexes.get(tblCls, {}).get('fields', []) + sql_indexes['__common__']['fields']
tbl_fields = sql_indexes.get(tblCls, {}).get('fields', []) + sql_indexes['__common__']['fields']

tbl_data = self.dbUtils.spanner.exec_sql('SELECT * FROM {} LIMIT 1'.format(tblCls))

Expand Down Expand Up @@ -293,7 +292,7 @@ def create_indexes(self):
)
self.dbUtils.exec_rdbm_query(sql_cmd)
elif Config.rdbm_type == 'pgsql':
sql_cmd ='CREATE INDEX ON "{}" (({}));'.format(
sql_cmd ='CREATE INDEX ON "{}" {};'.format(
tblCls,
tmp_str.safe_substitute({'field':attr.name})
)
Expand Down Expand Up @@ -326,7 +325,7 @@ def create_indexes(self):
)
self.dbUtils.exec_rdbm_query(sql_cmd)
elif Config.rdbm_type == 'pgsql':
sql_cmd = 'CREATE INDEX ON "{}" ("{}");'.format(
sql_cmd = 'CREATE INDEX ON "{}" {};'.format(
tblCls,
custom_index
)
Expand Down Expand Up @@ -362,9 +361,12 @@ def import_ldif(self):
self.dbUtils.import_ldif(ldif_files)

def rdbmProperties(self):
if Config.rdbm_type in ('sql', 'mysql'):
if Config.rdbm_type in ('pgsql', 'mysql'):
Config.rdbm_password_enc = self.obscure(Config.rdbm_password)
self.renderTemplateInOut(Config.jansRDBMProperties, Config.templateFolder, Config.configFolder)
src_temp_fn = os.path.join(Config.templateFolder, 'jans-{}.properties'.format(Config.rdbm_type))
targtet_fn = os.path.join(Config.configFolder, Config.jansRDBMProperties)
rendered_tmp = self.render_template(src_temp_fn)
self.writeFile(targtet_fn, rendered_tmp)

elif Config.rdbm_type == 'spanner':
if Config.spanner_emulator_host:
Expand Down
12 changes: 12 additions & 0 deletions jans-linux-setup/jans_setup/setup_app/static.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,18 @@ class BackendTypes:
PGSQL = 4
SPANNER = 5

class BackendStrings:
LOCAL_OPENDJ = 'Local OpenDj'
REMOTE_OPENDJ = 'Remote OpenDj'
LOCAL_COUCHBASE = 'Local Couchbase'
REMOTE_COUCHBASE = 'Remote Couchbase'
LOCAL_MYSQL = 'Local MySQL'
REMOTE_MYSQL = 'Remote MySQL'
CLOUD_SPANNER = 'Cloud Spanner'
SAPNNER_EMULATOR = 'Spanner Emulator'
LOCAL_PGSQL = 'Local PgSQL'
REMOTE_PGSQL = 'Remote PgSQL'

class AppType:
APPLICATION = 1
SERVICE = 2
Expand Down
2 changes: 2 additions & 0 deletions jans-linux-setup/jans_setup/setup_app/utils/collect_properties.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ def collect(self):
Config.rdbm_password_enc = jans_sql_prop['auth.userPassword']
Config.rdbm_password = self.unobscure(Config.rdbm_password_enc)
Config.rdbm_db = jans_sql_prop['db.schema.name']
if Config.rdbm_type == 'postgresql':
Config.rdbm_type = 'pgsql'

if not Config.persistence_type in ('couchbase', 'ldap') and Config.get('jansSpannerProperties') and os.path.exists(Config.jansSpannerProperties):
Config.rdbm_type = 'spanner'
Expand Down
41 changes: 26 additions & 15 deletions jans-linux-setup/jans_setup/setup_app/utils/db_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def bind(self, use_ssl=True, force=False):
for group in Config.mapping_locations:
if Config.mapping_locations[group] == 'rdbm':
if Config.rdbm_type in ('mysql', 'pgsql'):
base.logIt("Making MySql Conncetion")
base.logIt("Making {} Conncetion".format(Config.rdbm_type))
result = self.mysqlconnection()
if not result[0]:
print("{}FATAL: {}{}".format(colors.FAIL, result[1], colors.ENDC))
Expand Down Expand Up @@ -131,7 +131,7 @@ def sqlconnection(self, log=True):

@property
def json_dialects_instance(self):
return sqlalchemy.dialects.mysql.json.JSON if Config.rdbm_type == 'mysql' else sqlalchemy.dialects.postgresql.json.JSON
return sqlalchemy.dialects.mysql.json.JSON if Config.rdbm_type == 'mysql' else sqlalchemy.dialects.postgresql.json.JSONB

def mysqlconnection(self, log=True):
return self.sqlconnection(log)
Expand All @@ -151,7 +151,7 @@ def read_jans_schema(self, others=[]):

for attr in attribDataTypes.listAttributes:
if not attr in self.sql_data_types:
self.sql_data_types[attr] = { 'mysql': {'type': 'JSON'}, 'spanner': {'type': 'ARRAY<STRING(MAX)>'} }
self.sql_data_types[attr] = { 'mysql': {'type': 'JSON'}, 'pgsql': {'type': 'JSONB'}, 'spanner': {'type': 'ARRAY<STRING(MAX)>'} }

def in_subtable(self, table, attr):
if table in self.sub_tables[Config.rdbm_type]:
Expand Down Expand Up @@ -355,7 +355,7 @@ def dn_exists_rdbm(self, dn, table):
return result
return
sqlalchemy_table = self.Base.classes[table].__table__
return self.session.query(sqlalchemy_table).filter(sqlalchemy_table).filter(sqlalchemy_table.columns.dn == dn).first()
return self.session.query(sqlalchemy_table).filter(sqlalchemy_table.columns.dn == dn).first()


def spanner_to_dict(self, data):
Expand Down Expand Up @@ -577,14 +577,19 @@ def add_client2script(self, script_inum, client_id):
else:
jansConfProperty = {'v': []}

for oxconfigprop in jansConfProperty['v']:
if oxconfigprop.get('value1') == 'allowed_clients' and not client_id in oxconfigprop['value2']:
ox_configuration_property_list = jansConfProperty['v'] if Config.rdbm_type == 'mysql' else jansConfProperty

for i, oxconfigprop in enumerate(ox_configuration_property_list[:]):
if isinstance(oxconfigprop, str):
oxconfigprop = json.loads(oxconfigprop)
if oxconfigprop.get('value1') == 'allowed_clients' and client_id not in oxconfigprop['value2']:
oxconfigprop['value2'] = self.add2strlist(client_id, oxconfigprop['value2'])
ox_configuration_property_list[i] = json.dumps(oxconfigprop)
break
else:
jansConfProperty['v'].append({'value1': 'allowed_clients', 'value2': client_id})
ox_configuration_property_list.append(json.dumps({'value1': 'allowed_clients', 'value2': client_id}))

sqlalchemyObj.jansConfProperty = jansConfProperty
sqlalchemyObj.jansConfProperty = jansConfProperty if BackendTypes.MYSQL else ox_configuration_property_list
self.session.commit()


Expand Down Expand Up @@ -683,11 +688,13 @@ def rdm_automapper(self, force=False):
self.Base = sqlalchemy.ext.automap.automap_base(metadata=self.metadata)
self.Base.prepare()


# fix JSON type for mariadb
for tbl in self.Base.classes:
for col in tbl.__table__.columns:
if isinstance(col.type, sqlalchemy.dialects.mysql.LONGTEXT) and col.comment.lower() == 'json':
col.type = sqlalchemy.dialects.mysql.json.JSON()
if Config.rdbm_type == 'mysql':
for tbl in self.Base.classes:
for col in tbl.__table__.columns:
if isinstance(col.type, sqlalchemy.dialects.mysql.LONGTEXT) and col.comment.lower() == 'json':
col.type = sqlalchemy.dialects.mysql.json.JSON()

base.logIt("Reflected tables {}".format(list(self.metadata.tables.keys())))

Expand Down Expand Up @@ -730,7 +737,7 @@ def get_rdbm_val(self, key, val, rdbm_type=None):

data_type = self.get_attr_sql_data_type(key)

if data_type in ('SMALLINT', 'BOOL'):
if data_type in ('SMALLINT', 'BOOL', 'BOOLEAN'):
if val[0].lower() in ('1', 'on', 'true', 'yes', 'ok'):
return 1 if data_type == 'SMALLINT' else True
return 0 if data_type == 'SMALLINT' else False
Expand All @@ -751,7 +758,7 @@ def get_rdbm_val(self, key, val, rdbm_type=None):

return json_data

if data_type == 'ARRAY<STRING(MAX)>':
if data_type in ('ARRAY<STRING(MAX)>', 'JSONB'):
return val

return val[0]
Expand Down Expand Up @@ -835,6 +842,10 @@ def import_ldif(self, ldif_files, bucket=None, force=None):
cur_val = copy.deepcopy(getattr(sqlalchObj, attribute))
for val_ in new_val:
cur_val['v'].append(val_)
if Config.rdbm_type == 'mysql':
cur_val['v'].append(val_)
else:
cur_val.append(val_)
setattr(sqlalchObj, attribute, cur_val)
else:
setattr(sqlalchObj, attribute, new_val[0])
Expand Down Expand Up @@ -888,7 +899,7 @@ def import_ldif(self, ldif_files, bucket=None, force=None):

for col in sqlalchCls.__table__.columns:
if isinstance(col.type, self.json_dialects_instance) and not col.name in vals:
vals[col.name] = {'v': []}
vals[col.name] = {'v': []} if Config.rdbm_type == 'mysql' else []

sqlalchObj = sqlalchCls()

Expand Down
77 changes: 46 additions & 31 deletions jans-linux-setup/jans_setup/setup_app/utils/properties_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,15 @@
import ssl
import re
import pymysql
import psycopg2
import inspect
import ldap3

from setup_app import paths
from setup_app.messages import msg
from setup_app.utils import base
from setup_app.utils.cbm import CBM
from setup_app.static import InstallTypes, colors
from setup_app.static import InstallTypes, colors, BackendStrings

from setup_app.config import Config
from setup_app.utils.setup_utils import SetupUtils
Expand Down Expand Up @@ -659,16 +660,18 @@ def prompt_for_rdbm(self):
def prompt_for_backend(self):
print('Chose Backend Type:')

backend_types = ['Local OpenDj',
'Remote OpenDj',
'Local MySQL',
'Remote MySQL',
]
backend_types = [
BackendStrings.LOCAL_OPENDJ,
BackendStrings.LOCAL_MYSQL,
BackendStrings.REMOTE_MYSQL,
BackendStrings.LOCAL_PGSQL,
BackendStrings.REMOTE_PGSQL,
]

if not os.path.exists(os.path.join(Config.install_dir, 'package')):
backend_types += ['Remote Couchbase', 'Cloud Spanner']
backend_types += [BackendStrings.REMOTE_COUCHBASE, BackendStrings.CLOUD_SPANNER]
if 'couchbase' in self.getBackendTypes():
backend_types.insert(2, 'Local Couchbase')
backend_types.insert(2, BackendStrings.LOCAL_COUCHBASE)

nlist = []
for i, btype in enumerate(backend_types):
Expand Down Expand Up @@ -697,7 +700,7 @@ def prompt_for_backend(self):

backend_type_str = backend_types[int(choice)-1]

if backend_type_str == 'Local OpenDj':
if backend_type_str == BackendStrings.LOCAL_OPENDJ:
Config.opendj_install = InstallTypes.LOCAL
ldapPass = Config.ldapPass or Config.admin_password or self.getPW(special='.*=!%&+/-')

Expand All @@ -712,7 +715,7 @@ def prompt_for_backend(self):
Config.ldapPass = ldapPass


elif backend_type_str == 'Remote OpenDj':
elif backend_type_str == BackendStrings.REMOTE_OPENDJ:
Config.opendj_install = InstallTypes.REMOTE
while True:
ldapHost = self.getPrompt(" LDAP hostname")
Expand All @@ -726,7 +729,7 @@ def prompt_for_backend(self):
Config.ldapPass = ldapPass
Config.ldap_hostname = ldapHost

elif backend_type_str == 'Local Couchbase':
elif backend_type_str == BackendStrings.LOCAL_COUCHBASE:
Config.opendj_install = InstallTypes.NONE
Config.cb_install = InstallTypes.LOCAL
Config.isCouchbaseUserAdmin = True
Expand All @@ -742,7 +745,7 @@ def prompt_for_backend(self):
Config.cb_password = cbPass
Config.mapping_locations = { group: 'couchbase' for group in Config.couchbaseBucketDict }

elif backend_type_str == 'Remote Couchbase':
elif backend_type_str == BackendStrings.REMOTE_COUCHBASE:
Config.opendj_install = InstallTypes.NONE
Config.cb_install = InstallTypes.REMOTE

Expand All @@ -756,38 +759,50 @@ def prompt_for_backend(self):

Config.mapping_locations = { group: 'couchbase' for group in Config.couchbaseBucketDict }

elif backend_type_str == 'Local MySQL':
elif backend_type_str in (BackendStrings.LOCAL_MYSQL, BackendStrings.LOCAL_PGSQL):
Config.opendj_install = InstallTypes.NONE
Config.rdbm_install = True
Config.rdbm_install_type = InstallTypes.LOCAL
Config.rdbm_type = 'mysql'
Config.rdbm_host = 'localhost'
Config.rdbm_user = 'jans'
Config.rdbm_password = self.getPW(special='.*=+-()[]{}')
Config.rdbm_port = 3306
Config.rdbm_db = 'jansdb'

elif backend_type_str == 'Remote MySQL':
if backend_type_str == BackendStrings.LOCAL_MYSQL:
Config.rdbm_port = 3306
Config.rdbm_type = 'mysql'
else:
Config.rdbm_port = 5432
Config.rdbm_type = 'pgsql'
if not Config.rdbm_host:

if not Config.rdbm_password:
Config.rdbm_password = self.getPW(special='.*=+-()[]{}')

elif backend_type_str in (BackendStrings.REMOTE_MYSQL, BackendStrings.REMOTE_PGSQL):
Config.opendj_install = InstallTypes.NONE
Config.rdbm_install = True
Config.rdbm_install_type = InstallTypes.REMOTE
Config.rdbm_type = 'mysql'
if backend_type_str == BackendStrings.REMOTE_MYSQL:
Config.rdbm_port = 3306
Config.rdbm_type = 'mysql'
else:
Config.rdbm_port = 5432
Config.rdbm_type = 'pgsql'

while True:
Config.rdbm_host = self.getPrompt(" Mysql host", Config.get('rdbm_host'))
Config.rdbm_port = self.getPrompt(" Mysql port", 3306, itype=int, indent=1)
Config.rdbm_user = self.getPrompt(" Mysql user", Config.get('rdbm_user'))
Config.rdbm_password = self.getPrompt(" Mysql password")
Config.rdbm_db = self.getPrompt(" Mysql database", Config.get('rdbm_db'))
Config.rdbm_host = self.getPrompt(" {} host".format(Config.rdbm_type.upper()), Config.get('rdbm_host'))
Config.rdbm_port = self.getPrompt(" {} port".format(Config.rdbm_type.upper()), Config.rdbm_port, itype=int, indent=1)
Config.rdbm_user = self.getPrompt(" {} user".format(Config.rdbm_type.upper()), Config.get('rdbm_user'))
Config.rdbm_password = self.getPrompt(" {} password".format(Config.rdbm_type.upper()))
Config.rdbm_db = self.getPrompt(" {} database".format(Config.rdbm_type.upper()), Config.get('rdbm_db'))

try:
pymysql.connect(host=Config.rdbm_host, user=Config.rdbm_user, password=Config.rdbm_password, database=Config.rdbm_db, port=Config.rdbm_port)
print(" {}MySQL connection was successful{}".format(colors.OKGREEN, colors.ENDC))
if Config.rdbm_type == 'mysql':
pymysql.connect(host=Config.rdbm_host, user=Config.rdbm_user, password=Config.rdbm_password, database=Config.rdbm_db, port=Config.rdbm_port)
else:
psycopg2.connect(dbname=Config.rdbm_db, user=Config.rdbm_user, password=Config.rdbm_password, host=Config.rdbm_host, port=Config.rdbm_port)
print(" {}{} connection was successful{}".format(colors.OKGREEN, Config.rdbm_type.upper(), colors.ENDC))
break
except Exception as e:
print(" {}Can't connect to MySQL: {}{}".format(colors.DANGER, e, colors.ENDC))
print(" {}Can't connect to {}: {}{}".format(colors.DANGER,Config.rdbm_type.upper(), e, colors.ENDC))

elif backend_type_str == 'Cloud Spanner':
elif backend_type_str == BackendStrings.CLOUD_SPANNER:
Config.opendj_install = InstallTypes.NONE
Config.rdbm_type = 'spanner'
Config.rdbm_install = True
Expand Down
Loading