From 19d1216634c13bd61051e5756482c20e7e7c406f Mon Sep 17 00:00:00 2001 From: suryapratapdesai Date: Tue, 22 Oct 2024 14:02:32 +0530 Subject: [PATCH 1/3] DSE-38906 Provision to take various ini files at runtime. This is most sought feature request by customers in cases of Shared Multi-user machines with different ini. files for each customer. --- cmlutils/project_entrypoint.py | 93 +++++++++++++++++++++++++--------- examples/batch_export.py | 6 ++- examples/batch_import.py | 17 +++++-- 3 files changed, 88 insertions(+), 28 deletions(-) diff --git a/cmlutils/project_entrypoint.py b/cmlutils/project_entrypoint.py index a5e0ce6..d385836 100644 --- a/cmlutils/project_entrypoint.py +++ b/cmlutils/project_entrypoint.py @@ -85,6 +85,13 @@ def project_cmd(): """ +@click.group(name="experimental") +def experimental_cmd(): + """ + Sub-entrypoint for WIP/Experimental commands + """ + + @project_cmd.command(name="export") @click.option( "--project_name", @@ -92,10 +99,17 @@ def project_cmd(): help="Name of the project to be migrated. Make sure the name matches with the section name in export-config.ini file", required=True, ) -def project_export_cmd(project_name): +@click.option( + "--ini_location", + "-l", + default="/.cmlutils/export-config.ini", + show_default=True, + help="filepath of the export-config.ini file. default is /.cmlutils/export-config.ini", +) +def project_export_cmd(project_name, ini_location): pexport = None config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/export-config.ini", project_name + os.path.expanduser("~") + ini_location, project_name ) username = config[USERNAME_KEY] @@ -207,17 +221,24 @@ def project_export_cmd(project_name): help="Name of the project to be migrated. Make sure the name matches with the section name in import-config.ini file", required=True, ) +@click.option( + "--ini_location", + "-l", + default="/.cmlutils/import-config.ini", + show_default=True, + help="filepath of the import-config.ini file. default is /.cmlutils/import-config.ini", +) @click.option( "--verify", "-v", - is_flag=True, + is_flag=False, help="Flag to automatically trigger migration validation after import.", ) -def project_import_cmd(project_name, verify): +def project_import_cmd(project_name, ini_location, verify=False): pimport = None import_diff_file_list = None config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/import-config.ini", project_name + os.path.expanduser("~") + ini_location, project_name ) username = config[USERNAME_KEY] @@ -299,11 +320,10 @@ def project_import_cmd(project_name, verify): ) start_time = time.time() if verify: - import_diff_file_list=pimport.transfer_project(log_filedir=log_filedir, verify=True) + import_diff_file_list = pimport.transfer_project(log_filedir=log_filedir, verify=True) else: pimport.transfer_project(log_filedir=log_filedir) - if uses_engine: proj_patch_metadata = {"default_project_engine_type": "legacy_engine"} pimport.convert_project_to_engine_based( @@ -340,7 +360,9 @@ def project_import_cmd(project_name, verify): pimport.terminate_ssh_session() # If verification is also needed after import if verify: - print("***************************************************** Started verifying migration for project: {} ***************************************************** ".format(project_name)) + print( + "***************************************************** Started verifying migration for project: {} ***************************************************** ".format( + project_name)) ( imported_project_data, imported_project_list, @@ -578,8 +600,9 @@ def project_import_cmd(project_name, verify): True if (job_diff or job_config_diff) else False, message="Job Verification", ) - result = [export_diff_file_list,import_diff_file_list,proj_diff, - proj_config_diff,app_diff,app_config_diff,model_diff,model_config_diff,job_diff, job_config_diff] + result = [export_diff_file_list, import_diff_file_list, proj_diff, + proj_config_diff, app_diff, app_config_diff, model_diff, model_config_diff, job_diff, + job_config_diff] migration_status = all(not sublist for sublist in result) validation_data["isMigrationSuccessful"] = migration_status update_verification_status( @@ -612,13 +635,29 @@ def project_import_cmd(project_name, verify): help="Name of project migrated. Make sure the name matches with the section name in import-config.ini and export-config.ini file", required=True, ) -def project_verify_cmd(project_name): +@click.option( + "--export_ini_location", + "-el", + default="/.cmlutils/export-config.ini", + show_default=True, + help="filepath of the export-config.ini file. default is /.cmlutils/export-config.ini", +) +@click.option( + "--import_ini_location", + "-il", + default="/.cmlutils/import-config.ini", + show_default=True, + help="filepath of the import-config.ini file. default is /.cmlutils/import-config.ini", +) +def project_verify_cmd(project_name, export_ini_location, import_ini_location): + pexport = None validation_data = dict() config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/export-config.ini", project_name + os.path.expanduser("~") + export_ini_location, project_name ) + export_username = config[USERNAME_KEY] export_url = config[URL_KEY] export_apiv1_key = config[API_V1_KEY] @@ -709,7 +748,7 @@ def project_verify_cmd(project_name): pexport.terminate_ssh_session() pimport = None import_config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/import-config.ini", project_name + os.path.expanduser("~") + import_ini_location, project_name ) import_username = import_config[USERNAME_KEY] @@ -742,8 +781,8 @@ def project_verify_cmd(project_name): for v in validators: validation_response = v.validate() if ( - validation_response.validation_status - == ValidationResponseStatus.FAILED + validation_response.validation_status + == ValidationResponseStatus.FAILED ): logging.error( "Validation error for project %s: %s", @@ -924,8 +963,9 @@ def project_verify_cmd(project_name): True if (job_diff or job_config_diff) else False, message="Job Verification", ) - result = [export_diff_file_list,import_diff_file_list,proj_diff, - proj_config_diff,app_diff,app_config_diff,model_diff,model_config_diff,job_diff, job_config_diff] + result = [export_diff_file_list, import_diff_file_list, proj_diff, + proj_config_diff, app_diff, app_config_diff, model_diff, model_config_diff, job_diff, + job_config_diff] migration_status = all(not sublist for sublist in result) update_verification_status( not migration_status, @@ -961,11 +1001,18 @@ def project_helpers_cmd(): """ +@click.option( + "--ini_location", + "-l", + default="/.cmlutils/import-config.ini", + show_default=True, + help="filepath of the import-config.ini file. default is /.cmlutils/import-config.ini", +) @project_helpers_cmd.command("populate_engine_runtimes_mapping") -def populate_engine_runtimes_mapping(): +def populate_engine_runtimes_mapping(ini_location): project_name = "DEFAULT" config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/import-config.ini", project_name + os.path.expanduser("~") + ini_location, project_name ) username = config[USERNAME_KEY] @@ -1020,10 +1067,10 @@ def populate_engine_runtimes_mapping(): # Please make sure utility is having necessary permissions to write/overwrite data try: with open( - os.path.expanduser("~") - + "/.cmlutils/" - + "legacy_engine_runtime_constants.json", - "w", + os.path.expanduser("~") + + "/.cmlutils/" + + "legacy_engine_runtime_constants.json", + "w", ) as legacy_engine_runtime_constants: dump(legacy_runtime_image_map, legacy_engine_runtime_constants) except: diff --git a/examples/batch_export.py b/examples/batch_export.py index c805f36..174e484 100644 --- a/examples/batch_export.py +++ b/examples/batch_export.py @@ -33,8 +33,12 @@ def _get_project_list(file_path: str): def main(): + + # Change the ini location as per the convenience + ini_location = "/.cmlutils/export-config.ini" + project_names = _get_project_list( - os.path.expanduser("~") + "/.cmlutils/export-config.ini" + os.path.expanduser("~") + ini_location ) print(project_names) project_iter = [] diff --git a/examples/batch_import.py b/examples/batch_import.py index d94b75f..92cf8f3 100644 --- a/examples/batch_import.py +++ b/examples/batch_import.py @@ -16,6 +16,8 @@ # This variable controls if user want to trigger migration validation automatically after import # NOTE: Migration validation is resource intensive task keep the BATCH_SIZE to optimal size + +# DO NOT ENABLE this flag VERIFY = False @@ -39,7 +41,10 @@ def get_absolute_path(path: str) -> str: def import_validate(project_name: str): - output_dir = _read_config_file((os.path.expanduser("~") + "/.cmlutils/import-config.ini"), + # Change the ini location as per the convenience + ini_location = "/.cmlutils/import-config.ini" + + output_dir = _read_config_file((os.path.expanduser("~") + ini_location), project_name) import_metrics_file_path = os.path.join(get_absolute_path(output_dir[OUTPUT_DIR_KEY]), project_name, IMPORT_METRIC_FILE) @@ -85,9 +90,12 @@ def _get_project_list(file_path: str): def main(): - failed_validation_list = list() + + # Change the ini location as per the convenience + ini_location = "/.cmlutils/import-config.ini" + project_names = _get_project_list( - os.path.expanduser("~") + "/.cmlutils/import-config.ini" + os.path.expanduser("~") + ini_location ) project_iter = [] @@ -100,8 +108,9 @@ def main(): # call a function on each item in a list pool.starmap(import_project, project_iter) - # validation summary if VERIFY=True + # Please DO NOT enable this feature. if VERIFY: + failed_validation_list = list() for project in project_names: result = import_validate(project) if not result: From a2cbbea88fa729753fe449a09abbc07d0f765bc7 Mon Sep 17 00:00:00 2001 From: gmalik Date: Thu, 14 Nov 2024 13:09:09 +0530 Subject: [PATCH 2/3] DSE-38906: Pulling collborator changes --- cmlutils/constants.py | 2 + cmlutils/directory_utils.py | 5 + cmlutils/project_entrypoint.py | 151 +++++++++++++++++++++++-- cmlutils/projects.py | 196 +++++++++++++++++++++++++++++++-- cmlutils/utils.py | 33 ++++++ 5 files changed, 370 insertions(+), 17 deletions(-) diff --git a/cmlutils/constants.py b/cmlutils/constants.py index b05ad9c..8f64511 100644 --- a/cmlutils/constants.py +++ b/cmlutils/constants.py @@ -57,6 +57,8 @@ class ApiV1Endpoints(Enum): RUNTIMES = "/api/v1/runtimes" USER_INFO = "/api/v1/users/$username" PROJECTS_SUMMARY = "/api/v1/users/$username/projects-summary?all=true&context=$username&sortColumn=updated_at&projectName=$projectName&limit=$limit&offset=$offset" + COLLABORATORS = "/api/v2/projects/$project_id/collaborators?page_size=$page_size&page_token=$page_token" + ADD_COLLABORATOR = "/api/v2/projects/$project_id/collaborators/$user_name" """Mapping of old fields v1 to new fields of v2""" diff --git a/cmlutils/directory_utils.py b/cmlutils/directory_utils.py index 8ad2f51..4cb3113 100644 --- a/cmlutils/directory_utils.py +++ b/cmlutils/directory_utils.py @@ -16,6 +16,11 @@ def get_project_metadata_file_path(top_level_dir: str, project_name: str) -> str "project-metadata.json", ) +def get_project_collaborators_file_path(top_level_dir: str, project_name: str) -> str: + return os.path.join( + get_project_metadata_dir_path(top_level_dir, project_name), + "project-collaborators.json", + ) def get_models_metadata_file_path(top_level_dir: str, project_name: str) -> str: return os.path.join( diff --git a/cmlutils/project_entrypoint.py b/cmlutils/project_entrypoint.py index d385836..9fced3e 100644 --- a/cmlutils/project_entrypoint.py +++ b/cmlutils/project_entrypoint.py @@ -22,11 +22,12 @@ from cmlutils.script_models import ValidationResponseStatus from cmlutils.utils import ( compare_metadata, + compare_collaborator_metadata, get_absolute_path, parse_runtimes_v2, read_json_file, update_verification_status, - write_json_file + write_json_file, ) from cmlutils.validator import ( initialize_export_validators, @@ -136,7 +137,7 @@ def project_export_cmd(project_name, ini_location): project_slug=project_name, owner_type="", ) - creator_username, project_slug, owner_type = pobj.get_creator_username() + creator_username, project_slug, owner_type, public_id = pobj.get_creator_username() if creator_username is None: logging.error( "Validation error: Cannot find project - %s under username %s", @@ -181,6 +182,7 @@ def project_export_cmd(project_name, ini_location): ) start_time = time.time() pexport.transfer_project_files(log_filedir=log_filedir) + pexport.set_project_public_id(public_id=public_id) exported_data = pexport.dump_project_and_related_metadata() print("\033[32m✔ Export of Project {} Successful \033[0m".format(project_name)) print( @@ -199,6 +201,12 @@ def project_export_cmd(project_name, ini_location): exported_data.get("application_name_list"), ) ) + print( + "\033[34m\tExported {} Collaborators {}\033[0m".format( + exported_data.get("total_collaborators"), + exported_data.get("collaborator_list"), + ) + ) end_time = time.time() export_file = log_filedir + constants.EXPORT_METRIC_FILE write_json_file(file_path=export_file, json_data=exported_data) @@ -349,6 +357,12 @@ def project_import_cmd(project_name, ini_location, verify=False): import_data.get("application_name_list"), ) ) + print( + "\033[34m\tImported {} Collaborators {}\033[0m".format( + import_data.get("total_collaborators"), + import_data.get("collaborator_list"), + ) + ) end_time = time.time() import_file = log_filedir + constants.IMPORT_METRIC_FILE write_json_file(file_path=import_file, json_data=import_data) @@ -372,6 +386,8 @@ def project_import_cmd(project_name, ini_location, verify=False): imported_app_list, imported_job_data, imported_job_list, + imported_collaborator_data, + imported_collaborator_list, ) = pimport.collect_imported_project_data(project_id=project_id) # import_diff_file_list = pimport.verify_project(log_filedir=log_filedir) @@ -394,7 +410,7 @@ def project_import_cmd(project_name, ini_location, verify=False): _configure_project_command_logging(log_filedir, project_name) import_file = log_filedir + constants.IMPORT_METRIC_FILE - with open(import_file, 'r') as file: + with open(import_file, "r") as file: validation_data = json.load(file) try: # Get username of the creator of project - This is required so that admins can also migrate the project @@ -412,6 +428,7 @@ def project_import_cmd(project_name, ini_location, verify=False): export_creator_username, export_project_slug, export_owner_type, + _ ) = pobj.get_creator_username() if export_creator_username is None: logging.error( @@ -464,6 +481,8 @@ def project_import_cmd(project_name, ini_location, verify=False): exported_app_list, exported_job_data, exported_job_list, + exported_collaborator_data, + exported_collaborator_list, ) = pexport.collect_export_project_data() # File verification @@ -600,9 +619,58 @@ def project_import_cmd(project_name, ini_location, verify=False): True if (job_diff or job_config_diff) else False, message="Job Verification", ) - result = [export_diff_file_list, import_diff_file_list, proj_diff, - proj_config_diff, app_diff, app_config_diff, model_diff, model_config_diff, job_diff, - job_config_diff] + # Collaborators verification + collaborator_diff, collaborator_config_diff = ( + compare_collaborator_metadata( + imported_collaborator_data, + exported_collaborator_data, + imported_collaborator_list, + exported_collaborator_list, + skip_field=None, + ) + ) + logging.info( + "Source collaborator list {}".format(exported_collaborator_list) + ) + logging.info( + "Destination collaborator list {}".format( + imported_collaborator_list + ) + ) + logging.info( + "All collaborators in source project is present at destination project ".format( + collaborator_diff + ) + if not collaborator_diff + else "collaborator {} Not Found in source or destination".format( + collaborator_diff + ) + ) + logging.info( + "No collaborator Config Difference Found" + if not collaborator_config_diff + else "Difference in collaborator Config {}".format( + collaborator_config_diff + ) + ) + update_verification_status( + True if (collaborator_diff or collaborator_config_diff) else False, + message="Collaborator Verification", + ) + result = [ + export_diff_file_list, + import_diff_file_list, + proj_diff, + proj_config_diff, + app_diff, + app_config_diff, + model_diff, + model_config_diff, + job_diff, + job_config_diff, + collaborator_diff, + collaborator_config_diff, + ] migration_status = all(not sublist for sublist in result) validation_data["isMigrationSuccessful"] = migration_status update_verification_status( @@ -672,7 +740,7 @@ def project_verify_cmd(project_name, export_ini_location, import_ini_location): logging.info("Started Verifying project: %s", project_name) import_file = log_filedir + constants.IMPORT_METRIC_FILE try: - with open(import_file, 'r') as file: + with open(import_file, "r") as file: validation_data = json.load(file) except: logging.error("File not found Exception: ", exc_info=1) @@ -692,6 +760,7 @@ def project_verify_cmd(project_name, export_ini_location, import_ini_location): export_creator_username, export_project_slug, export_owner_type, + _ ) = pobj.get_creator_username() if export_creator_username is None: logging.error( @@ -744,6 +813,8 @@ def project_verify_cmd(project_name, export_ini_location, import_ini_location): exported_app_list, exported_job_data, exported_job_list, + exported_collaborator_data, + exported_collaborator_list, ) = pexport.collect_export_project_data() pexport.terminate_ssh_session() pimport = None @@ -825,6 +896,8 @@ def project_verify_cmd(project_name, export_ini_location, import_ini_location): imported_app_list, imported_job_data, imported_job_list, + imported_collaborator_data, + imported_collaborator_list, ) = pimport.collect_imported_project_data(project_id=project_id) # File verification @@ -963,9 +1036,67 @@ def project_verify_cmd(project_name, export_ini_location, import_ini_location): True if (job_diff or job_config_diff) else False, message="Job Verification", ) - result = [export_diff_file_list, import_diff_file_list, proj_diff, - proj_config_diff, app_diff, app_config_diff, model_diff, model_config_diff, job_diff, - job_config_diff] + result = [ + export_diff_file_list, + import_diff_file_list, + proj_diff, + proj_config_diff, + app_diff, + app_config_diff, + model_diff, + model_config_diff, + job_diff, + job_config_diff, + ] + + # Collaborators verification + collaborator_diff, collaborator_config_diff = compare_collaborator_metadata( + imported_collaborator_data, + exported_collaborator_data, + imported_collaborator_list, + exported_collaborator_list, + skip_field=None, + ) + logging.info( + "Source collaborator list {}".format(exported_collaborator_list) + ) + logging.info( + "Destination collaborator list {}".format(imported_collaborator_list) + ) + logging.info( + "All collaborators in source project is present at destination project ".format( + collaborator_diff + ) + if not collaborator_diff + else "collaborator {} Not Found in source or destination".format( + collaborator_diff + ) + ) + logging.info( + "No collaborator Config Difference Found" + if not collaborator_config_diff + else "Difference in collaborator Config {}".format( + collaborator_config_diff + ) + ) + update_verification_status( + True if (collaborator_diff or collaborator_config_diff) else False, + message="Collaborator Verification", + ) + result = [ + export_diff_file_list, + import_diff_file_list, + proj_diff, + proj_config_diff, + app_diff, + app_config_diff, + model_diff, + model_config_diff, + job_diff, + job_config_diff, + collaborator_diff, + collaborator_config_diff, + ] migration_status = all(not sublist for sublist in result) update_verification_status( not migration_status, diff --git a/cmlutils/projects.py b/cmlutils/projects.py index 11d83c1..a76cd87 100644 --- a/cmlutils/projects.py +++ b/cmlutils/projects.py @@ -22,6 +22,7 @@ get_models_metadata_file_path, get_project_data_dir_path, get_project_metadata_file_path, + get_project_collaborators_file_path ) from cmlutils.ssh import open_ssh_endpoint from cmlutils.utils import ( @@ -287,6 +288,7 @@ def __init__( self.owner_type = owner_type super().__init__(host, username, project_name, api_key, ca_path, project_slug) self.metrics_data = dict() + self.project_public_id = None # Get CDSW project info using API v1 def get_project_infov1(self): @@ -339,10 +341,10 @@ def get_creator_username(self): ) """ - End loop - a. If response len is less than MAX_API_PAGE_LENGTH + End loop + a. If response len is less than MAX_API_PAGE_LENGTH => Possible if less number of records - => Possible if response is [] => len 0 + => Possible if response is [] => len 0 b. If length of response is greater than MAX_API_PAGE_LENGTH => If source is CDSW, as CDSW doesn't honor limit c. If CDSW non-paginated response length is exactly the MAX_API_PAGE_LENGTH """ @@ -364,14 +366,16 @@ def get_creator_username(self): project["owner"]["username"], project["slug_raw"], constants.ORGANIZATION_TYPE, + project["public_identifier"], ) else: return ( project["creator"]["username"], project["slug_raw"], constants.USER_TYPE, + project["public_identifier"], ) - return None, None, None + return None, None, None, None # Get all models list info using API v1 def get_models_listv1(self, project_id: int): @@ -437,6 +441,9 @@ def get_model_infov1(self, model_id: str): ) return response.json() + def set_project_public_id(self, public_id: str): + self.project_public_id = public_id + # Get Job info using API v1 def get_job_infov1(self, job_id: int): endpoint = Template(ApiV1Endpoints.JOB_INFO.value).substitute( @@ -605,6 +612,25 @@ def verify_project_files(self, log_filedir: str): self.terminate_ssh_session() return result + def get_project_collaborators_v2(self, page_token: str, project_id: str): + endpoint = Template(ApiV2Endpoints.COLLABORATORS.value).substitute( + page_size=constants.MAX_API_PAGE_LENGTH, + page_token=page_token, + project_id=project_id, + ) + + response = call_api_v2( + host=self.host, + endpoint=endpoint, + method="GET", + user_token=self.apiv2_key, + ca_path=self.ca_path, + ) + result_list = response.json() + if result_list: + return result_list + return None + def _export_project_metadata(self): filepath = get_project_metadata_file_path( top_level_dir=self.top_level_dir, project_name=self.project_name @@ -638,6 +664,32 @@ def _export_project_metadata(self): self.project_id = project_info_resp["id"] write_json_file(file_path=filepath, json_data=project_metadata) + def _export_project_collaborators(self): + filepath = get_project_collaborators_file_path( + top_level_dir=self.top_level_dir, project_name=self.project_name + ) + logging.info("Exporting project collaborators to path %s", filepath) + project_collaborators_resp = self.get_project_collaborators_v2( + project_id=self.project_public_id, page_token="" + ) + project_collaborators_new = {"collaborators": []} + + usernames = [] + + for collaborator in project_collaborators_resp["collaborators"]: + collaborator_entry = { + "permission": collaborator["permission"], + "username": collaborator["user"]["username"], + } + project_collaborators_new["collaborators"].append(collaborator_entry) + usernames.append(collaborator["user"]["username"]) + + self.metrics_data["total_collaborators"] = len( + project_collaborators_new["collaborators"] + ) + self.metrics_data["collaborator_list"] = sorted(usernames) + write_json_file(file_path=filepath, json_data=project_collaborators_new) + def _export_models_metadata(self): filepath = get_models_metadata_file_path( top_level_dir=self.top_level_dir, project_name=self.project_name @@ -794,6 +846,29 @@ def collect_export_application_list(self): app_metadata_list.append(app_metadata) return app_metadata_list, sorted(app_name_list) + def collect_export_collaborator_list(self, project_id): + project_collaborators_resp = self.get_project_collaborators_v2( + project_id=project_id, page_token="" + ) + project_collaborators_new = {"collaborators": []} + + usernames = [] + + for collaborator in project_collaborators_resp["collaborators"]: + collaborator_entry = { + "permission": collaborator["permission"], + "username": collaborator["user"]["username"], + } + project_collaborators_new["collaborators"].append(collaborator_entry) + usernames.append(collaborator["user"]["username"]) + + self.metrics_data["total_collaborators"] = len( + project_collaborators_new["collaborators"] + ) + self.metrics_data["collaborator_list"] = sorted(usernames) + + return project_collaborators_new["collaborators"], sorted(usernames) + def _export_job_metadata(self): filepath = get_jobs_metadata_file_path( top_level_dir=self.top_level_dir, project_name=self.project_name @@ -876,6 +951,7 @@ def dump_project_and_related_metadata(self): self._export_models_metadata() self._export_application_metadata() self._export_job_metadata() + self._export_project_collaborators() return self.metrics_data def collect_export_project_data(self): @@ -891,6 +967,9 @@ def collect_export_project_data(self): ) app_data, app_list = self.collect_export_application_list() job_data, job_list = self.collect_export_job_list() + collaborators_data, collaborators_list = self.collect_export_collaborator_list( + proj_data_raw["public_identifier"] + ) return ( proj_data, proj_list, @@ -900,6 +979,8 @@ def collect_export_project_data(self): app_list, job_data, job_list, + collaborators_data, + collaborators_list, ) @@ -942,10 +1023,10 @@ def get_creator_username(self): ) """ - End loop - a. If response len is less than MAX_API_PAGE_LENGTH + End loop + a. If response len is less than MAX_API_PAGE_LENGTH => Possible if less number of records - => Possible if response is [] => len 0 + => Possible if response is [] => len 0 b. If length of response is greater than MAX_API_PAGE_LENGTH => If source is CDSW, as CDSW doesn't honor limit c. If CDSW non-paginated response length is exactly the MAX_API_PAGE_LENGTH """ @@ -1130,6 +1211,22 @@ def create_model_build_v2( ) return + def add_proj_collaborator_v2(self, proj_id: str, user_name: str, metadata): + try: + endpoint = Template(ApiV2Endpoints.ADD_COLLABORATOR.value).substitute( + project_id=proj_id, user_name=user_name + ) + call_api_v2( + host=self.host, + endpoint=endpoint, + method="PUT", + user_token=self.apiv2_key, + json_data=metadata, + ca_path=self.ca_path, + ) + except KeyError as e: + raise + def create_application_v2(self, proj_id: str, app_metadata) -> str: try: endpoint = Template(ApiV2Endpoints.CREATE_APP.value).substitute( @@ -1420,10 +1517,18 @@ def import_metadata(self, project_id: str): self.create_paused_jobs( project_id=project_id, job_metadata_filepath=job_metadata_filepath ) + proj_collaborator_filepath = get_project_collaborators_file_path( + top_level_dir=self.top_level_dir, project_name=self.project_name + ) + self.add_project_collaborators( + project_id=project_id, + collaborator_metadata_filepath=proj_collaborator_filepath, + ) self.get_project_infov2(proj_id=project_id) self.collect_import_model_list(project_id=project_id) self.collect_import_application_list(project_id=project_id) self.collect_import_job_list(project_id=project_id) + self.collect_import_collaborator_list(project_id=project_id) return self.metrics_data def collect_imported_project_data(self, project_id: str): @@ -1438,6 +1543,7 @@ def collect_imported_project_data(self, project_id: str): model_data, model_list = self.collect_import_model_list(project_id=project_id) app_data, app_list = self.collect_import_application_list(project_id=project_id) job_data, job_list = self.collect_import_job_list(project_id=project_id) + collaborator_data, collaborators = self.collect_import_collaborator_list(project_id=project_id) return ( proj_data, proj_list, @@ -1447,6 +1553,8 @@ def collect_imported_project_data(self, project_id: str): app_list, job_data, job_list, + collaborator_data, + collaborators, ) def create_models(self, project_id: str, models_metadata_filepath: str): @@ -1588,6 +1696,38 @@ def create_stoppped_applications(self, project_id: str, app_metadata_filepath: s logging.error(f"Error: {e}") raise + def add_project_collaborators( + self, project_id: str, collaborator_metadata_filepath: str + ): + try: + collaborator_metadata = read_json_file(collaborator_metadata_filepath) + collaborator_metadata_list = collaborator_metadata.get("collaborators") + + if collaborator_metadata_list != None: + for collaborator_metadata in collaborator_metadata_list: + try: + self.add_proj_collaborator_v2( + proj_id=project_id, + user_name=collaborator_metadata["username"], + metadata=collaborator_metadata, + ) + except Exception as e: + logging.error( + f"Failed to add collaborator {collaborator_metadata['username']}. Error: {e}" + ) + else: + logging.info( + f"{collaborator_metadata['username']} has been added successfully as a collaborator." + ) + return + except FileNotFoundError as e: + logging.info("No collaborator-metadata file found for migration") + return + except Exception as e: + logging.error("Collaborator migration failed") + logging.error(f"Error: {e}") + raise + def create_paused_jobs(self, project_id: str, job_metadata_filepath: str): try: runtime_list = self.get_all_runtimes() @@ -1689,6 +1829,25 @@ def get_project_infov2(self, proj_id: str): ) return response.json() + def get_project_collaborators_v2(self, page_token: str, project_id: str): + endpoint = Template(ApiV2Endpoints.COLLABORATORS.value).substitute( + page_size=constants.MAX_API_PAGE_LENGTH, + page_token=page_token, + project_id=project_id, + ) + + response = call_api_v2( + host=self.host, + endpoint=endpoint, + method="GET", + user_token=self.apiv2_key, + ca_path=self.ca_path, + ) + result_list = response.json() + if result_list: + return result_list + return None + def collect_import_job_list(self, project_id): job_list = self.get_jobs_listv2(proj_id=project_id)["jobs"] job_name_list = [] @@ -1754,3 +1913,26 @@ def collect_import_application_list(self, project_id): self.metrics_data["total_application"] = len(app_name_list) self.metrics_data["application_name_list"] = sorted(app_name_list) return app_metadata_list, sorted(app_name_list) + + def collect_import_collaborator_list(self, project_id): + project_collaborators_resp = self.get_project_collaborators_v2( + project_id=project_id, page_token="" + ) + project_collaborators_new = {"collaborators": []} + + usernames = [] + + for collaborator in project_collaborators_resp["collaborators"]: + collaborator_entry = { + "permission": collaborator["permission"], + "username": collaborator["user"]["username"], + } + project_collaborators_new["collaborators"].append(collaborator_entry) + usernames.append(collaborator["user"]["username"]) + + self.metrics_data["total_collaborators"] = len( + project_collaborators_new["collaborators"] + ) + self.metrics_data["collaborator_list"] = sorted(usernames) + + return project_collaborators_new["collaborators"], sorted(usernames) \ No newline at end of file diff --git a/cmlutils/utils.py b/cmlutils/utils.py index 2bc4c67..3ab35a7 100644 --- a/cmlutils/utils.py +++ b/cmlutils/utils.py @@ -320,6 +320,39 @@ def compare_metadata( config_differences[name]= difference return data_list_diff, config_differences +def compare_collaborator_metadata( + import_data, export_data, import_data_list, export_data_list, skip_field=None +): + if skip_field is None: + skip_field = [] + + data_list_diff = list(set(sorted(export_data_list)) - set(sorted(import_data_list))) + config_differences = {} + + import_data_dict = {data["username"]: data for data in import_data} + export_data_dict = {data["username"]: data for data in export_data} + + for name, im_data in import_data_dict.items(): + ex_data = export_data_dict.get(name) + + if ex_data is None: + continue + + for key, value in im_data.items(): + if key not in skip_field: + ex_value = ex_data.get(key) + if ex_value is not None and str(ex_value) != str(value): + difference = [ + "{} value in destination is {}, and source is {}".format( + key, str(value), str(ex_value) + ) + ] + if config_differences.get(name): + config_differences[name].extend(difference) + else: + config_differences[name] = difference + return data_list_diff, config_differences + def update_verification_status(data_diff, message): if data_diff: From a16d8afcec222161394bffa572120e012f9009f6 Mon Sep 17 00:00:00 2001 From: gmalik Date: Thu, 14 Nov 2024 14:50:40 +0530 Subject: [PATCH 3/3] DSE-38906: Corrected endpoint to right enum for Api V2 --- cmlutils/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmlutils/constants.py b/cmlutils/constants.py index 8f64511..ffdb0a4 100644 --- a/cmlutils/constants.py +++ b/cmlutils/constants.py @@ -41,6 +41,8 @@ class ApiV2Endpoints(Enum): SEARCH_APP = "/api/v2/projects/$project_id/applications?search_filter=$search_option&page_size=100000" RUNTIME_ADDONS = "/api/v2/runtimeaddons?search_filter=$search_option" RUNTIMES = "/api/v2/runtimes?page_size=$page_size&page_token=$page_token" + COLLABORATORS = "/api/v2/projects/$project_id/collaborators?page_size=$page_size&page_token=$page_token" + ADD_COLLABORATOR = "/api/v2/projects/$project_id/collaborators/$user_name" class ApiV1Endpoints(Enum): @@ -57,8 +59,6 @@ class ApiV1Endpoints(Enum): RUNTIMES = "/api/v1/runtimes" USER_INFO = "/api/v1/users/$username" PROJECTS_SUMMARY = "/api/v1/users/$username/projects-summary?all=true&context=$username&sortColumn=updated_at&projectName=$projectName&limit=$limit&offset=$offset" - COLLABORATORS = "/api/v2/projects/$project_id/collaborators?page_size=$page_size&page_token=$page_token" - ADD_COLLABORATOR = "/api/v2/projects/$project_id/collaborators/$user_name" """Mapping of old fields v1 to new fields of v2"""