-
Notifications
You must be signed in to change notification settings - Fork 39
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Feature/multi model artifact handler #869
Changes from 12 commits
6d9fa4a
6a930f1
5f3b316
802e438
35e8464
e625107
74a235e
848972e
d35b917
3ac5338
822c7e8
3d4d950
d4ef023
c40ea8b
d0309f4
19ec921
9089028
464cb37
49be8f8
a2894a5
5d327cd
b7e7d74
4418bf8
e2e7099
4cc8823
7f3cc7f
32f4826
9f69e35
ad0ce2e
eec2b4d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,288 @@ | ||
import json | ||
import ads.common | ||
import oci | ||
import os | ||
import ads | ||
from ads.model.datascience_model import DataScienceModel | ||
from typing import List, Optional | ||
import logging | ||
|
||
logger = logging.getLogger("ads.model_description") | ||
logger.setLevel(logging.INFO) | ||
|
||
|
||
class DataScienceModelCollection(DataScienceModel): | ||
VipulMascarenhas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
def _auth(self): | ||
VipulMascarenhas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
""" | ||
Internal method that authenticates the model description instance by initializing OCI clients. | ||
|
||
Parameters: | ||
- None | ||
|
||
Returns: | ||
- None | ||
|
||
Note: | ||
- This method retrieves authentication data using default signer from the `ads.common.auth` module. | ||
- The region information is extracted from the authentication data. | ||
""" | ||
authData = ads.common.auth.default_signer() | ||
signer = authData["signer"] | ||
self.region = authData["config"]["region"] | ||
|
||
# data science client | ||
self.data_science_client = oci.data_science.DataScienceClient( | ||
{"region": self.region}, signer=signer | ||
) | ||
# oss client | ||
self.object_storage_client = oci.object_storage.ObjectStorageClient( | ||
{"region": self.region}, signer=signer | ||
) | ||
|
||
def __init__(self, spec: ads.Dict = None, **kwargs) -> None: | ||
super().__init__(spec, **kwargs) | ||
|
||
self.empty_json = { | ||
"version": "1.0", | ||
"type": "modelOSSReferenceDescription", | ||
"models": [], | ||
} | ||
self.region = "" | ||
self._auth() | ||
|
||
self.set_spec(self.CONST_MODEL_FILE_DESCRIPTION, self.empty_json) | ||
|
||
def with_ref_model_id(self, model_ocid: str): | ||
VipulMascarenhas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
# if model given then get that as the starting reference point | ||
logger.info("Getting model details from backend") | ||
try: | ||
get_model_artifact_content_response = ( | ||
self.data_science_client.get_model_artifact_content( | ||
model_id=model_ocid, | ||
) | ||
) | ||
content = get_model_artifact_content_response.data.content | ||
self.set_spec(self.CONST_MODEL_FILE_DESCRIPTION, json.loads(content)) | ||
except json.JSONDecodeError as e: | ||
logger.error(f"Error decoding JSON: {e}") | ||
raise e | ||
except Exception as e: | ||
logger.error(f"An unexpected error occurred: {e}") | ||
raise e | ||
return self | ||
|
||
def add( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's rename this to add_artifact(uri:str, files=Optional[List[str]]) which takes in uri and files list as input to be consistent with how paths are referred to within ads. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The user won't have uri handy, hence we have parameters like namespace, and bucket to make it easy.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. isn't the uri same as |
||
self, | ||
namespace: str, | ||
bucket: str, | ||
prefix: Optional[str] = None, | ||
files: Optional[List[str]] = None, | ||
): | ||
""" | ||
Adds information about objects in a specified bucket to the model description JSON. | ||
|
||
Parameters: | ||
- namespace (str): The namespace of the object storage. | ||
- bucket (str): The name of the bucket containing the objects. | ||
- prefix (str, optional): The prefix used to filter objects within the bucket. Defaults to None. | ||
- files (list of str, optional): A list of file names to include in the model description. | ||
If provided, only objects with matching file names will be included. Defaults to None. | ||
|
||
Returns: | ||
- None | ||
|
||
Raises: | ||
- ValueError: If no files are found to add to the model description. | ||
|
||
Note: | ||
- If `files` is not provided, it retrieves information about all objects in the bucket. | ||
If `files` is provided, it only retrieves information about objects with matching file names. | ||
- If no objects are found to add to the model description, a ValueError is raised. | ||
""" | ||
|
||
# Remove if the model already exists | ||
self.remove(namespace, bucket, prefix) | ||
|
||
def check_if_file_exists(fileName): | ||
VipulMascarenhas marked this conversation as resolved.
Show resolved
Hide resolved
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please double check if the existing utils methods are already implemented such functionality. |
||
isExists = False | ||
try: | ||
headResponse = self.object_storage_client.head_object( | ||
namespace, bucket, object_name=fileName | ||
) | ||
if headResponse.status == 200: | ||
isExists = True | ||
except Exception as e: | ||
if hasattr(e, "status") and e.status == 404: | ||
logger.error(f"File not found in bucket: {fileName}") | ||
else: | ||
logger.error(f"An error occured: {e}") | ||
return isExists | ||
|
||
# Function to un-paginate the api call with while loop | ||
def list_obj_versions_unpaginated(): | ||
objectStorageList = [] | ||
has_next_page, opc_next_page = True, None | ||
while has_next_page: | ||
response = self.object_storage_client.list_object_versions( | ||
namespace_name=namespace, | ||
bucket_name=bucket, | ||
prefix=prefix, | ||
fields="name,size", | ||
page=opc_next_page, | ||
) | ||
objectStorageList.extend(response.data.items) | ||
has_next_page = response.has_next_page | ||
opc_next_page = response.next_page | ||
return objectStorageList | ||
|
||
# Fetch object details and put it into the objects variable | ||
objectStorageList = [] | ||
if files == None: | ||
objectStorageList = list_obj_versions_unpaginated() | ||
else: | ||
for fileName in files: | ||
if check_if_file_exists(fileName=fileName): | ||
objectStorageList.append( | ||
self.object_storage_client.list_object_versions( | ||
namespace_name=namespace, | ||
bucket_name=bucket, | ||
prefix=fileName, | ||
fields="name,size", | ||
).data.items[0] | ||
) | ||
|
||
objects = [ | ||
{"name": obj.name, "version": obj.version_id, "sizeInBytes": obj.size} | ||
for obj in objectStorageList | ||
if obj.size > 0 | ||
] | ||
|
||
if len(objects) == 0: | ||
error_message = ( | ||
f"No files to add in the bucket: {bucket} with namespace: {namespace} " | ||
f"and prefix: {prefix}. File names: {files}" | ||
) | ||
logger.error(error_message) | ||
raise ValueError(error_message) | ||
|
||
tmp_model_file_description = self.model_file_description | ||
tmp_model_file_description["models"].append( | ||
{ | ||
"namespace": namespace, | ||
"bucketName": bucket, | ||
"prefix": prefix, | ||
"objects": objects, | ||
} | ||
) | ||
self.set_spec(self.CONST_MODEL_FILE_DESCRIPTION, tmp_model_file_description) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we modify If content is passed, then no need to create the dict within that function.
In this case The inputs for files isn't ideal, it might be good to implement a dataclass for this and pass a
This isn't urgent though, we can update this later on. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for this suggestion. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sure, let's mark a todo somewhere to make sure of this. |
||
|
||
def remove(self, namespace: str, bucket: str, prefix: Optional[str] = None): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same as add_artifact - rename this to remove_artifact(uri:str) which takes in uri to be consistent with how paths are referred to within ads. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes done. |
||
""" | ||
Removes information about objects in a specified bucket from the model description JSON. | ||
|
||
Parameters: | ||
- namespace (str): The namespace of the object storage. | ||
- bucket (str): The name of the bucket containing the objects. | ||
- prefix (str, optional): The prefix used to filter objects within the bucket. Defaults to None. | ||
|
||
Returns: | ||
- None | ||
|
||
Note: | ||
- This method removes information about objects in the specified bucket from the | ||
instance of the ModelDescription. | ||
- If a matching model (with the specified namespace, bucket name, and prefix) is found | ||
in the model description JSON, it is removed. | ||
- If no matching model is found, the method returns without making any changes. | ||
""" | ||
|
||
def findModelIdx(): | ||
for idx, model in enumerate(self.model_file_description["models"]): | ||
VipulMascarenhas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if ( | ||
model["namespace"], | ||
model["bucketName"], | ||
(model["prefix"] if ("prefix" in model) else None), | ||
) == (namespace, bucket, prefix): | ||
return idx | ||
return -1 | ||
|
||
modelSearchIdx = findModelIdx() | ||
if modelSearchIdx == -1: | ||
return | ||
else: | ||
# model found case | ||
self.model_file_description["models"].pop(modelSearchIdx) | ||
|
||
def create(self): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this isn't needed since we can just the existing There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes removed this. |
||
""" | ||
Saves the model to the Model Catalog of Oracle Cloud Infrastructure (OCI) Data Science service. | ||
|
||
Parameters: | ||
- project_ocid (str): The OCID (Oracle Cloud Identifier) of the OCI Data Science project. | ||
- compartment_ocid (str): The OCID of the compartment in which the model will be created. | ||
- display_name (str, optional): The display name for the created model. If not provided, | ||
a default display name indicating the creation timestamp is used. Defaults to None. | ||
|
||
Returns: | ||
- str: The OCID of the created model. | ||
|
||
Note: | ||
- The display name defaults to a string indicating the creation timestamp if not provided. | ||
""" | ||
tmp_file_path = self.build() | ||
self = self.with_artifact(uri=tmp_file_path) | ||
created_model = super().create() | ||
try: | ||
os.remove(tmp_file_path) | ||
except Exception as e: | ||
logger.error(f"Error occurred while cleaning file: {e}") | ||
raise e | ||
mrDzurb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
return created_model.id | ||
|
||
def build(self) -> str: | ||
VipulMascarenhas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
""" | ||
Builds the model description JSON and writes it to a file. | ||
|
||
Parameters: | ||
- None | ||
|
||
Returns: | ||
- str: The absolute file path where the model description JSON is stored. | ||
|
||
Note: | ||
- This method serializes the current model description attribute to a JSON file named 'resultModelDescription.json' with an indentation of 2 spaces. | ||
""" | ||
logger.info("Building...") | ||
file_path = "resultModelDescription.json" | ||
mrDzurb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
try: | ||
with open(file_path, "w") as json_file: | ||
json.dump(self.model_file_description, json_file, indent=2) | ||
except IOError as e: | ||
logger.error( | ||
f"Error writing to file '{file_path}': {e}" | ||
) # Handle the exception accordingly, e.g., log the error, retry writing, etc. | ||
except Exception as e: | ||
logger.error( | ||
f"An unexpected error occurred: {e}" | ||
) # Handle other unexpected exceptions | ||
logger.info("Model Artifact stored successfully.") | ||
return os.path.abspath(file_path) | ||
|
||
def show(self) -> str: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is covered by There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes removing this. |
||
""" | ||
Displays the current model description JSON in a human-readable format. | ||
|
||
Parameters: | ||
- None | ||
|
||
Returns: | ||
- str: The json representation of current model artifact | ||
|
||
Note: | ||
- The JSON representation of the model description is formatted with an indentation | ||
of 4 spaces. | ||
""" | ||
logger.info(json.dumps(self.model_file_description, indent=4)) | ||
return json.dumps(self.model_file_description, indent=4) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Wouldn't this be better -
logger = logging.getLogger(__name__)
?