feat: jans-linux-setup spanner rest client (#3436)

* feat: jans-linux-setup spanner rest client * fix: jans-linux-setup check spanner connection * fix: jans-linux-setup real spanner server support * fix: jans-linux-setup real spanner creds property name * fix: jans-linux-setup code smells
JanssenProject · Jan 12, 2023 · e4d1d0c · e4d1d0c
1 parent e20e817
commit e4d1d0c
Show file tree

Hide file tree

Showing 9 changed files with 307 additions and 231 deletions.
diff --git a/jans-linux-setup/jans_setup/app_info.json b/jans-linux-setup/jans_setup/app_info.json
@@ -6,11 +6,11 @@
   "JYTHON_VERSION": "2.7.3",
   "OPENDJ_VERSION": "4.4.12",
   "JANS_MAVEN": "https://jenkins.jans.io",
-  "EXTERNAL_LIBS": "https://ox.gluu.org/icrby8xcvbcv/",
   "APPLE_WEBAUTHN": "https://www.apple.com/certificateauthority/Apple_WebAuthn_Root_CA.pem",
   "SQLALCHEMY": "https://github.com/sqlalchemy/sqlalchemy/archive/rel_1_3_23.zip",
-  "PYJWT": "https://github.com/jpadilla/pyjwt/archive/refs/tags/2.3.0.zip",
+  "PYJWT": "https://github.com/jpadilla/pyjwt/archive/refs/tags/2.6.0.zip",
   "PROMPT_TOOLKIT": "https://github.com/prompt-toolkit/python-prompt-toolkit/archive/refs/tags/3.0.33.zip",
   "WCWIDTH": "https://github.com/jquast/wcwidth/archive/refs/tags/0.2.5.zip",
-  "PYGMENTS": "https://github.com/pygments/pygments/archive/refs/tags/2.13.0.zip"
+  "PYGMENTS": "https://github.com/pygments/pygments/archive/refs/tags/2.13.0.zip",
+  "CRYPTOGRAPHY": "https://files.pythonhosted.org/packages/7a/46/8b58d6b8244ff613ecb983b9428d1168dd0b014a34e13fb19737b9ba1fc1/cryptography-39.0.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
 }
diff --git a/jans-linux-setup/jans_setup/setup_app/downloads.py b/jans-linux-setup/jans_setup/setup_app/downloads.py
@@ -19,68 +19,43 @@ def download_jans_acrhieve():
             verbose=True
             )
 
-def get_grpcio_package(data):
 
-    pyversion = 'cp{0}{1}'.format(sys.version_info.major, sys.version_info.minor)
-
-    package = {}
-
-    for package_ in data['urls']:
-
-        if package_['python_version'] == pyversion and 'manylinux' in package_['filename'] and package_['filename'].endswith('x86_64.whl'):
-            if package_['upload_time'] > package.get('upload_time',''):
-                package = package_
-                break
-
-    return package
-
-def download_gcs():
-    gcs_dir = os.path.join(base.pylib_dir, 'gcs')
+def download_sqlalchemy():
+    sqlalchemy_dir = os.path.join(base.pylib_dir, 'sqlalchemy')
 
-    if os.path.exists(gcs_dir) and not base.argsp.force_download:
+    if os.path.exists(sqlalchemy_dir) and not base.argsp.force_download:
         return
 
-    base.logIt("Downloading Spanner modules")
-    gcs_download_url = os.path.join(base.current_app.app_info['EXTERNAL_LIBS'], 'spanner/gcs.tgz')
-
     with tempfile.TemporaryDirectory() as tmp_dir:
+        sqlalchemy_zip_file = os.path.join(tmp_dir, os.path.basename(base.current_app.app_info['SQLALCHEMY']))
+        base.download(base.current_app.app_info['SQLALCHEMY'], sqlalchemy_zip_file, verbose=True)
+        base.extract_subdir(sqlalchemy_zip_file, 'lib/sqlalchemy', sqlalchemy_dir)
 
-        target_fn = os.path.join(tmp_dir, 'gcs.tgz')
-        base.download(gcs_download_url, target_fn, verbose=True)
-        shutil.unpack_archive(target_fn, base.pylib_dir)
-
-        grpcio_fn = os.path.join(tmp_dir, 'grpcio_fn.json')
-        base.download('https://pypi.org/pypi/grpcio/1.46.0/json', grpcio_fn, verbose=True)
-        data = base.readJsonFile(grpcio_fn)
-
-        package = get_grpcio_package(data)
-
-        if package.get('url'):
-            target_whl_fn = os.path.join(tmp_dir, os.path.basename(package['url']))
-            base.download(package['url'], target_whl_fn, verbose=True)
-            whl_zip = zipfile.ZipFile(target_whl_fn)
 
-            for member in  whl_zip.filelist:
-                whl_zip.extract(member, gcs_dir)
+def download_cryptography():
+    cryptography_dir = os.path.join(base.pylib_dir, 'cryptography')
 
-            whl_zip.close()
+    if os.path.exists(cryptography_dir) and not base.argsp.force_download:
+        return
 
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        cryptography_zip_file = os.path.join(tmp_dir, os.path.basename(base.current_app.app_info['CRYPTOGRAPHY']))
+        base.download(base.current_app.app_info['CRYPTOGRAPHY'], cryptography_zip_file, verbose=True)
+        base.extract_subdir(cryptography_zip_file, 'cryptography', cryptography_dir, par_dir='')
 
-def download_sqlalchemy():
-    sqlalchemy_dir = os.path.join(base.pylib_dir, 'sqlalchemy')
+def download_pyjwt():
+    pyjwt_dir = os.path.join(base.pylib_dir, 'jwt')
 
-    if os.path.exists(sqlalchemy_dir) and not base.argsp.force_download:
+    if os.path.exists(pyjwt_dir) and not base.argsp.force_download:
         return
 
     with tempfile.TemporaryDirectory() as tmp_dir:
-        sqlalchemy_zip_file = os.path.join(tmp_dir, os.path.basename(base.current_app.app_info['SQLALCHEMY']))
-        base.download(base.current_app.app_info['SQLALCHEMY'], sqlalchemy_zip_file, verbose=True)
-        base.extract_subdir(sqlalchemy_zip_file, 'lib/sqlalchemy', sqlalchemy_dir)
-
+        pyjwt_dir_zip_file = os.path.join(tmp_dir, os.path.basename(base.current_app.app_info['PYJWT']))
+        base.download(base.current_app.app_info['PYJWT'], pyjwt_dir_zip_file, verbose=True)
+        base.extract_subdir(pyjwt_dir_zip_file, 'jwt', pyjwt_dir)
 
 def download_all():
     download_files = []
-    sys.path.insert(0, os.path.join(base.pylib_dir, 'gcs'))
     modules = glob.glob(os.path.join(base.ces_dir, 'installers/*.py'))
 
     for installer in modules:
@@ -101,6 +76,6 @@ def download_all():
 
 def download_apps():
     download_jans_acrhieve()
-    if base.current_app.profile == 'jans':
-        download_gcs()
     download_sqlalchemy()
+    download_cryptography()
+    download_pyjwt()
diff --git a/jans-linux-setup/jans_setup/setup_app/installers/rdbm.py b/jans-linux-setup/jans_setup/setup_app/installers/rdbm.py
@@ -252,7 +252,7 @@ def create_tables(self, jans_schema_files):
                 sql_cmd = alter_table_sql_cmd.format(attr['sql']['add_table'], col_def)
 
                 if Config.rdbm_type == 'spanner':
-                    req = self.dbUtils.spanner.create_table(sql_cmd.strip(';'))
+                    req = self.dbUtils.spanner_client.exec_sql(sql_cmd.strip(';'))
                 else:
                     self.dbUtils.exec_rdbm_query(sql_cmd)
                 tables.append(sql_cmd)
@@ -265,9 +265,9 @@ def create_subtables(self):
             for sattr, sdt in self.dbUtils.sub_tables[Config.rdbm_type][subtable]:
                 subtable_columns = []
                 sql_cmd = 'CREATE TABLE `{0}_{1}` (`doc_id` STRING(64) NOT NULL, `dict_doc_id` STRING(64), `{1}` {2}) PRIMARY KEY (`doc_id`, `dict_doc_id`), INTERLEAVE IN PARENT `{0}` ON DELETE CASCADE'.format(subtable, sattr, sdt)
-                self.dbUtils.spanner.create_table(sql_cmd)
+                self.dbUtils.spanner_client.exec_sql(sql_cmd)
                 sql_cmd_index = 'CREATE INDEX `{0}_{1}Idx` ON `{0}_{1}` (`{1}`)'.format(subtable, sattr)
-                self.dbUtils.spanner.create_table(sql_cmd_index)
+                self.dbUtils.spanner_client.exec_sql(sql_cmd_index)
 
 
     def get_index_name(self, attrname):
@@ -292,11 +292,11 @@ def create_indexes(self):
                         sql_indexes['__common__']['fields'].append(attr_name)
 
         if Config.rdbm_type == 'spanner':
-            tables = self.dbUtils.spanner.get_tables()
+            tables = self.dbUtils.spanner_client.get_tables()
             for tblCls in tables:
                 tbl_fields = sql_indexes.get(tblCls, {}).get('fields', []) + sql_indexes['__common__']['fields']
 
-                tbl_data = self.dbUtils.spanner.exec_sql('SELECT * FROM {} LIMIT 1'.format(tblCls))
+                tbl_data = self.dbUtils.spanner_client.exec_sql('SELECT * FROM {} LIMIT 1'.format(tblCls))
 
                 for attr in tbl_data.get('fields', []):
                     if attr['name'] == 'doc_id':
@@ -315,15 +315,15 @@ def create_indexes(self):
                                     tblCls,
                                     attr_name
                                 )
-                        self.dbUtils.spanner.create_table(sql_cmd)
+                        self.dbUtils.spanner_client.exec_sql(sql_cmd)
 
                 for i, custom_index in enumerate(sql_indexes.get(tblCls, {}).get('custom', [])):
                     sql_cmd = 'CREATE INDEX `{0}_CustomIdx{1}` ON {0} ({2})'.format(
                                     tblCls,
                                     i+1, 
                                     custom_index
                                 )
-                    self.dbUtils.spanner.create_table(sql_cmd)
+                    self.dbUtils.spanner_client.exec_sql(sql_cmd)
 
         else:
             for tblCls in self.dbUtils.Base.classes.keys():
@@ -434,7 +434,7 @@ def rdbmProperties(self):
             else:
                 auth_cred_target_fn = os.path.join(Config.configFolder, 'google_application_credentials.json')
                 shutil.copy(Config.google_application_credentials, auth_cred_target_fn)
-                Config.templateRenderingDict['spanner_creds'] = 'auth.credentials-file={}'.format(auth_cred_target_fn)
+                Config.templateRenderingDict['spanner_creds'] = 'connection.credentials-file={}'.format(auth_cred_target_fn)
 
             self.renderTemplateInOut(Config.jansSpannerProperties, Config.templateFolder, Config.configFolder)
 

diff --git a/jans-linux-setup/jans_setup/setup_app/utils/base.py b/jans-linux-setup/jans_setup/setup_app/utils/base.py
@@ -383,10 +383,11 @@ def extract_from_zip(zip_file, sub_dir, target_dir, remove_target_dir=False):
                 target_fn.write_bytes(zipobj.read(member))
     zipobj.close()
 
-def extract_subdir(zip_fn, sub_dir, target_dir):
-    zip_obj = zipfile.ZipFile(zip_fn, 'r')
-    par_dir = zip_obj.namelist()[0]
-    zip_obj.close()
+def extract_subdir(zip_fn, sub_dir, target_dir, par_dir=None):
+    if par_dir is None:
+        zip_obj = zipfile.ZipFile(zip_fn, 'r')
+        par_dir = zip_obj.namelist()[0]
+        zip_obj.close()
 
     if not sub_dir.endswith('/'):
         sub_dir += '/'
@@ -397,7 +398,7 @@ def extract_subdir(zip_fn, sub_dir, target_dir):
         shutil.rmtree(target_dir_path)
 
     with tempfile.TemporaryDirectory() as unpack_dir:
-        shutil.unpack_archive(zip_fn, unpack_dir)
+        shutil.unpack_archive(zip_fn, unpack_dir, format='zip')
         shutil.copytree(os.path.join(unpack_dir, par_dir, sub_dir), target_dir)
 
 current_app.app_info = readJsonFile(os.path.join(par_dir, 'app_info.json'))