Skip to content

With this Change, added the collaborators migration support to the cmlutil. #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

With this Change, added the collaborators migration support to the cmlutil. #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmlutils/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ class ApiV2Endpoints(Enum):
SEARCH_APP = "/api/v2/projects/$project_id/applications?search_filter=$search_option&page_size=100000"
RUNTIME_ADDONS = "/api/v2/runtimeaddons?search_filter=$search_option"
RUNTIMES = "/api/v2/runtimes?page_size=$page_size&page_token=$page_token"
COLLABORATORS = "/api/v2/projects/$project_id/collaborators?page_size=$page_size&page_token=$page_token"
ADD_COLLABORATOR = "/api/v2/projects/$project_id/collaborators/$user_name"


class ApiV1Endpoints(Enum):
Expand Down
7 changes: 7 additions & 0 deletions cmlutils/directory_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@ def get_project_metadata_file_path(top_level_dir: str, project_name: str) -> str
)


def get_project_collaborators_file_path(top_level_dir: str, project_name: str) -> str:
return os.path.join(
get_project_metadata_dir_path(top_level_dir, project_name),
"project-collaborators.json",
)


def get_models_metadata_file_path(top_level_dir: str, project_name: str) -> str:
return os.path.join(
get_project_metadata_dir_path(top_level_dir, project_name),
Expand Down
198 changes: 172 additions & 26 deletions cmlutils/project_entrypoint.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@
from cmlutils.script_models import ValidationResponseStatus
from cmlutils.utils import (
compare_metadata,
compare_collaborator_metadata,
get_absolute_path,
parse_runtimes_v2,
read_json_file,
update_verification_status,
write_json_file
write_json_file,
)
from cmlutils.validator import (
initialize_export_validators,
Expand Down Expand Up @@ -122,7 +123,9 @@ def project_export_cmd(project_name):
project_slug=project_name,
owner_type="",
)
creator_username, project_slug, owner_type = pobj.get_creator_username()
creator_username, project_slug, owner_type, public_id = (
pobj.get_creator_username()
)
if creator_username is None:
logging.error(
"Validation error: Cannot find project - %s under username %s",
Expand Down Expand Up @@ -167,6 +170,7 @@ def project_export_cmd(project_name):
)
start_time = time.time()
pexport.transfer_project_files(log_filedir=log_filedir)
pexport.set_project_public_id(public_id=public_id)
exported_data = pexport.dump_project_and_related_metadata()
print("\033[32m✔ Export of Project {} Successful \033[0m".format(project_name))
print(
Expand All @@ -185,6 +189,12 @@ def project_export_cmd(project_name):
exported_data.get("application_name_list"),
)
)
print(
"\033[34m\tExported {} Collaborators {}\033[0m".format(
exported_data.get("total_collaborators"),
exported_data.get("collaborator_list"),
)
)
end_time = time.time()
export_file = log_filedir + constants.EXPORT_METRIC_FILE
write_json_file(file_path=export_file, json_data=exported_data)
Expand Down Expand Up @@ -299,11 +309,12 @@ def project_import_cmd(project_name, verify):
)
start_time = time.time()
if verify:
import_diff_file_list=pimport.transfer_project(log_filedir=log_filedir, verify=True)
import_diff_file_list = pimport.transfer_project(
log_filedir=log_filedir, verify=True
)
else:
pimport.transfer_project(log_filedir=log_filedir)


if uses_engine:
proj_patch_metadata = {"default_project_engine_type": "legacy_engine"}
pimport.convert_project_to_engine_based(
Expand All @@ -329,6 +340,12 @@ def project_import_cmd(project_name, verify):
import_data.get("application_name_list"),
)
)
print(
"\033[34m\tImported {} Collaborators {}\033[0m".format(
import_data.get("total_collaborators"),
import_data.get("collaborator_list"),
)
)
end_time = time.time()
import_file = log_filedir + constants.IMPORT_METRIC_FILE
write_json_file(file_path=import_file, json_data=import_data)
Expand All @@ -340,7 +357,11 @@ def project_import_cmd(project_name, verify):
pimport.terminate_ssh_session()
# If verification is also needed after import
if verify:
print("***************************************************** Started verifying migration for project: {} ***************************************************** ".format(project_name))
print(
"***************************************************** Started verifying migration for project: {} ***************************************************** ".format(
project_name
)
)
(
imported_project_data,
imported_project_list,
Expand All @@ -350,6 +371,8 @@ def project_import_cmd(project_name, verify):
imported_app_list,
imported_job_data,
imported_job_list,
imported_collaborator_data,
imported_collaborator_list,
) = pimport.collect_imported_project_data(project_id=project_id)
# import_diff_file_list = pimport.verify_project(log_filedir=log_filedir)

Expand All @@ -372,7 +395,7 @@ def project_import_cmd(project_name, verify):
_configure_project_command_logging(log_filedir, project_name)

import_file = log_filedir + constants.IMPORT_METRIC_FILE
with open(import_file, 'r') as file:
with open(import_file, "r") as file:
validation_data = json.load(file)
try:
# Get username of the creator of project - This is required so that admins can also migrate the project
Expand All @@ -386,11 +409,9 @@ def project_import_cmd(project_name, verify):
project_slug=project_name,
owner_type="",
)
(
export_creator_username,
export_project_slug,
export_owner_type,
) = pobj.get_creator_username()
(export_creator_username, export_project_slug, export_owner_type, _) = (
pobj.get_creator_username()
)
if export_creator_username is None:
logging.error(
"Validation error: Cannot find project - %s under username %s",
Expand All @@ -410,7 +431,10 @@ def project_import_cmd(project_name, verify):
)
for v in validators:
validation_response = v.validate()
if validation_response.validation_status == ValidationResponseStatus.FAILED:
if (
validation_response.validation_status
== ValidationResponseStatus.FAILED
):
logging.error(
"Validation error: %s",
project_name,
Expand Down Expand Up @@ -442,6 +466,8 @@ def project_import_cmd(project_name, verify):
exported_app_list,
exported_job_data,
exported_job_list,
exported_collaborator_data,
exported_collaborator_list,
) = pexport.collect_export_project_data()

# File verification
Expand Down Expand Up @@ -506,7 +532,9 @@ def project_import_cmd(project_name, verify):
skip_field=["environment"],
)
logging.info("Source Application list {}".format(exported_app_list))
logging.info("Destination Application list {}".format(imported_app_list))
logging.info(
"Destination Application list {}".format(imported_app_list)
)
logging.info(
"All Application in source project is present at destination project ".format(
app_diff
Expand Down Expand Up @@ -540,7 +568,9 @@ def project_import_cmd(project_name, verify):
model_diff
)
if not model_diff
else "Model {} Not Found in source or destination".format(model_diff)
else "Model {} Not Found in source or destination".format(
model_diff
)
)
logging.info(
"No Model Config Difference Found"
Expand Down Expand Up @@ -578,13 +608,66 @@ def project_import_cmd(project_name, verify):
True if (job_diff or job_config_diff) else False,
message="Job Verification",
)
result = [export_diff_file_list,import_diff_file_list,proj_diff,
proj_config_diff,app_diff,app_config_diff,model_diff,model_config_diff,job_diff, job_config_diff]

# Collaborators verification
collaborator_diff, collaborator_config_diff = (
compare_collaborator_metadata(
imported_collaborator_data,
exported_collaborator_data,
imported_collaborator_list,
exported_collaborator_list,
skip_field=None,
)
)
logging.info(
"Source collaborator list {}".format(exported_collaborator_list)
)
logging.info(
"Destination collaborator list {}".format(
imported_collaborator_list
)
)
logging.info(
"All collaborators in source project is present at destination project ".format(
collaborator_diff
)
if not collaborator_diff
else "collaborator {} Not Found in source or destination".format(
collaborator_diff
)
)
logging.info(
"No collaborator Config Difference Found"
if not collaborator_config_diff
else "Difference in collaborator Config {}".format(
collaborator_config_diff
)
)
update_verification_status(
True if (collaborator_diff or collaborator_config_diff) else False,
message="Collaborator Verification",
)
result = [
export_diff_file_list,
import_diff_file_list,
proj_diff,
proj_config_diff,
app_diff,
app_config_diff,
model_diff,
model_config_diff,
job_diff,
job_config_diff,
collaborator_diff,
collaborator_config_diff,
]
migration_status = all(not sublist for sublist in result)
validation_data["isMigrationSuccessful"] = migration_status
update_verification_status(
not migration_status,
message="Migration Validation status for project : {} is".format(project_name),
message="Migration Validation status for project : {} is".format(
project_name
),
)
write_json_file(file_path=import_file, json_data=validation_data)

Expand Down Expand Up @@ -633,7 +716,7 @@ def project_verify_cmd(project_name):
logging.info("Started Verifying project: %s", project_name)
import_file = log_filedir + constants.IMPORT_METRIC_FILE
try:
with open(import_file, 'r') as file:
with open(import_file, "r") as file:
validation_data = json.load(file)
except:
logging.error("File not found Exception: ", exc_info=1)
Expand All @@ -649,11 +732,9 @@ def project_verify_cmd(project_name):
project_slug=project_name,
owner_type="",
)
(
export_creator_username,
export_project_slug,
export_owner_type,
) = pobj.get_creator_username()
(export_creator_username, export_project_slug, export_owner_type, _) = (
pobj.get_creator_username()
)
if export_creator_username is None:
logging.error(
"Validation error: Cannot find project - %s under username %s",
Expand Down Expand Up @@ -705,6 +786,8 @@ def project_verify_cmd(project_name):
exported_app_list,
exported_job_data,
exported_job_list,
exported_collaborator_data,
exported_collaborator_list,
) = pexport.collect_export_project_data()
pexport.terminate_ssh_session()
pimport = None
Expand Down Expand Up @@ -786,6 +869,8 @@ def project_verify_cmd(project_name):
imported_app_list,
imported_job_data,
imported_job_list,
imported_collaborator_data,
imported_collaborator_list,
) = pimport.collect_imported_project_data(project_id=project_id)

# File verification
Expand Down Expand Up @@ -924,12 +1009,73 @@ def project_verify_cmd(project_name):
True if (job_diff or job_config_diff) else False,
message="Job Verification",
)
result = [export_diff_file_list,import_diff_file_list,proj_diff,
proj_config_diff,app_diff,app_config_diff,model_diff,model_config_diff,job_diff, job_config_diff]
result = [
export_diff_file_list,
import_diff_file_list,
proj_diff,
proj_config_diff,
app_diff,
app_config_diff,
model_diff,
model_config_diff,
job_diff,
job_config_diff,
]

# Collaborators verification
collaborator_diff, collaborator_config_diff = compare_collaborator_metadata(
imported_collaborator_data,
exported_collaborator_data,
imported_collaborator_list,
exported_collaborator_list,
skip_field=None,
)
logging.info(
"Source collaborator list {}".format(exported_collaborator_list)
)
logging.info(
"Destination collaborator list {}".format(imported_collaborator_list)
)
logging.info(
"All collaborators in source project is present at destination project ".format(
collaborator_diff
)
if not collaborator_diff
else "collaborator {} Not Found in source or destination".format(
collaborator_diff
)
)
logging.info(
"No collaborator Config Difference Found"
if not collaborator_config_diff
else "Difference in collaborator Config {}".format(
collaborator_config_diff
)
)
update_verification_status(
True if (collaborator_diff or collaborator_config_diff) else False,
message="Collaborator Verification",
)
result = [
export_diff_file_list,
import_diff_file_list,
proj_diff,
proj_config_diff,
app_diff,
app_config_diff,
model_diff,
model_config_diff,
job_diff,
job_config_diff,
collaborator_diff,
collaborator_config_diff,
]
migration_status = all(not sublist for sublist in result)
update_verification_status(
not migration_status,
message="Migration Validation status for project : {} is".format(project_name),
message="Migration Validation status for project : {} is".format(
project_name
),
)
validation_data["isMigrationSuccessful"] = migration_status
write_json_file(file_path=import_file, json_data=validation_data)
Expand Down
Loading