-
Notifications
You must be signed in to change notification settings - Fork 6
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
added first iteration of easy_publish #153
Changes from all commits
828b678
c2af2f3
9c6041f
4e0a17f
e16cc99
4812942
9f299b8
4e58649
760b703
8e1d3c6
03e948f
dc9fb36
0524c89
1fb6615
3e573d2
d4e6232
981efe7
5ba2463
026ef66
3521090
af04041
802cfc0
ba9e7e7
0ecdbbc
bf55004
c93efeb
e35b097
fcbd173
624a59b
83c126f
ac7f1d7
873f34c
5e8d8c6
6b1af9c
1a77bfd
3b08006
dbc7126
71cb851
5e5d34c
9310f71
ee975d7
f4ec3c2
b1eac69
e8576bd
08d255a
f7eb374
ba0a4fe
7cbfded
7213660
21a2ea0
88dab48
a22de1d
a9ef0c3
62b77be
666f5c8
4828b88
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,7 +4,7 @@ formats: | |
|
||
python: | ||
version: 3.6 | ||
version: 3.8 | ||
install: | ||
- method: pip | ||
path: . | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,7 +2,7 @@ | |
import json | ||
import os | ||
from tempfile import mkstemp | ||
from typing import Union, Any, Optional, Tuple, Dict | ||
from typing import Sequence, Union, Any, Optional, Tuple, Dict, List | ||
import requests | ||
import globus_sdk | ||
|
||
|
@@ -14,6 +14,13 @@ | |
from funcx.sdk.client import FuncXClient | ||
from globus_sdk.scopes import AuthScopes, SearchScopes | ||
|
||
from dlhub_sdk.models.servables.keras import KerasModel | ||
from dlhub_sdk.models.servables.pytorch import TorchModel | ||
from dlhub_sdk.models.servables.python import PythonClassMethodModel | ||
from dlhub_sdk.models.servables.python import PythonStaticMethodModel | ||
from dlhub_sdk.models.servables.tensorflow import TensorFlowModel | ||
from dlhub_sdk.models.servables.sklearn import ScikitLearnModel | ||
|
||
from dlhub_sdk.config import DLHUB_SERVICE_ADDRESS, CLIENT_ID | ||
from dlhub_sdk.utils.futures import DLHubFuture | ||
from dlhub_sdk.utils.schemas import validate_against_dlhub_schema | ||
|
@@ -26,6 +33,10 @@ | |
logger = logging.getLogger(__name__) | ||
|
||
|
||
class HelpMessage(Exception): | ||
"""Raised from another error to provide the user an additional message""" | ||
|
||
|
||
class DLHubClient(BaseClient): | ||
"""Main class for interacting with the DLHub service | ||
|
||
|
@@ -343,6 +354,70 @@ def get_result(self, task_id, verbose=False): | |
result = result[0] | ||
return result | ||
|
||
def easy_publish(self, title: str, creators: Union[str, List[str]], short_name: str, servable_type: str, serv_options: Dict[str, Any], | ||
affiliations: List[Sequence[str]] = None, paper_doi: str = None): | ||
"""Simplified publishing method for servables | ||
|
||
Args: | ||
title (string): title for the servable | ||
creators (string | list): either the creator's name (FamilyName, GivenName) or a list of the creators' names | ||
short_name (string): shorthand name for the servable | ||
servable_type (string): the type of the servable, must be a member of ("static_method", | ||
"class_method", | ||
"keras", | ||
"pytorch", | ||
"tensorflow", | ||
"sklearn") more information on servable types can be found here: | ||
https://dlhub-sdk.readthedocs.io/en/latest/servable-types.html | ||
serv_options (dict): the servable_type specific arguments that are necessary for publishing | ||
affiliations (list): list of affiliations for each author | ||
paper_doi (str): DOI of a paper that describes the servable | ||
Returns: | ||
(string): task id of this submission, can be used to check for success | ||
Raises: | ||
ValueError: If the given servable_type is not in the list of acceptable types | ||
Exception: If the serv_options are incomplete or the request to publish results in an error | ||
""" | ||
# conversion table for model string names to classes | ||
models = {"static_method": PythonStaticMethodModel, | ||
"class_method": PythonClassMethodModel, | ||
"keras": KerasModel, | ||
"pytorch": TorchModel, | ||
"tensorflow": TensorFlowModel, | ||
"sklearn": ScikitLearnModel} | ||
|
||
# raise an error if the provided servable_type is invalid | ||
model = models.get(servable_type) | ||
if model is None: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if you wanted to shorten this a bit, you could do There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would, but I also need to store the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🤷♀️ I like "not in models" but not enough to change this here. |
||
raise ValueError(f"dl.easy_publish given invalid servable type: {servable_type}, please refer to the docstring") | ||
|
||
# attempt to construct the model and raise a helpful error if needed | ||
try: | ||
# if the servable is a python function, set the parameter to attempt to auto-generate the inputs/outputs | ||
if servable_type in {"static_method", "class_method"}: | ||
serv_options["auto_inspect"] = True | ||
|
||
model_info = model.create_model(**serv_options) | ||
except Exception as e: | ||
help_err = HelpMessage(f"Help can be found here:\nhttps://dlhub-sdk.readthedocs.io/en/latest/source/dlhub_sdk.models.servables.html#" | ||
f"{model.__module__}.{model.__name__}.create_model") | ||
raise help_err from e | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. just to clarify, this help_err won't eat the original exception message, correct? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No. specifying |
||
|
||
# set the required datacite fields | ||
model_info.set_title(title) | ||
creators = [creators] if isinstance(creators, str) else creators # handle the case where creators is a string | ||
model_info.set_creators(creators, affiliations or []) # affiliations if provided, else empty list | ||
model_info.set_name(short_name) | ||
|
||
if paper_doi is not None: | ||
model_info.add_related_resource(paper_doi, "DOI", "IsDescribedBy") | ||
|
||
# perform the publish | ||
task_id = self.publish_servable(model_info) | ||
|
||
# return the id of the publish task | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. great comments in this method Isaac, keep it up! 🎉 |
||
return task_id | ||
|
||
def publish_servable(self, model): | ||
"""Submit a servable to DLHub | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,8 +1,11 @@ | ||
"""Tools to annotate generic operations (e.g., class method calls) in Python""" | ||
import pickle as pkl | ||
import importlib | ||
from inspect import Signature | ||
|
||
from dlhub_sdk.models.servables import BaseServableModel, ArgumentTypeMetadata | ||
from dlhub_sdk.utils.types import compose_argument_block | ||
from dlhub_sdk.utils.inspect import signature_to_input, signature_to_output | ||
|
||
|
||
class BasePythonServableModel(BaseServableModel): | ||
|
@@ -113,14 +116,15 @@ class PythonClassMethodModel(BasePythonServableModel): | |
any arguments of the class that should be set as defaults.""" | ||
|
||
@classmethod | ||
def create_model(cls, path, method, function_kwargs=None) -> 'PythonClassMethodModel': | ||
def create_model(cls, path, method, function_kwargs=None, *, auto_inspect=False) -> 'PythonClassMethodModel': | ||
"""Initialize a model for a python object | ||
|
||
Args: | ||
path (string): Path to a pickled Python file | ||
method (string): Name of the method for this class | ||
function_kwargs (dict): Names and values of any other argument of the function to set | ||
the values must be JSON serializable. | ||
auto_inspect (boolean): Whether or not to attempt to automatically extract inputs from the function | ||
""" | ||
output = super(PythonClassMethodModel, cls).create_model(method, function_kwargs) | ||
|
||
|
@@ -135,6 +139,11 @@ def create_model(cls, path, method, function_kwargs=None) -> 'PythonClassMethodM | |
'class_name': class_name | ||
}) | ||
|
||
if auto_inspect: | ||
func = getattr(obj, method) | ||
|
||
output = add_extracted_metadata(func, output) | ||
|
||
return output | ||
|
||
def _get_handler(self): | ||
|
@@ -153,8 +162,8 @@ class PythonStaticMethodModel(BasePythonServableModel): | |
""" | ||
|
||
@classmethod | ||
def create_model(cls, module, method, autobatch=False, function_kwargs=None): | ||
"""Initialize the method | ||
def create_model(cls, module=None, method=None, autobatch=False, function_kwargs=None, *, f=None, auto_inspect=False): | ||
"""Initialize the method based on the provided arguments | ||
ascourtas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
Args: | ||
module (string): Name of the module holding the function | ||
|
@@ -163,28 +172,61 @@ def create_model(cls, module, method, autobatch=False, function_kwargs=None): | |
Calls :code:`map(f, list)` | ||
function_kwargs (dict): Names and values of any other argument of the function to set | ||
the values must be JSON serializable. | ||
f (object): function pointer to the desired python function | ||
auto_inspect (boolean): Whether or not to attempt to automatically extract inputs from the function | ||
Raises: | ||
TypeError: If there is no valid way to process the given arguments | ||
""" | ||
# if a pointer is provided, get the module and method | ||
if f is not None: | ||
ascourtas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
module, method = f.__module__, f.__name__ | ||
func = f | ||
# if it is not, ensure both the module and method are provided and get the function pointer | ||
elif module is not None and method is not None: | ||
module_obj = importlib.import_module(module) | ||
func = getattr(module_obj, method) | ||
else: | ||
raise TypeError("PythonStaticMethodModel.create_model was not provided valid arguments. Please provide either a funtion pointer" | ||
" or the module and name of the desired static function") | ||
|
||
output = super(PythonStaticMethodModel, cls).create_model(method, function_kwargs) | ||
|
||
output.servable.methods["run"].method_details.update({ | ||
'module': module, | ||
'autobatch': autobatch | ||
}) | ||
|
||
if auto_inspect: | ||
output = add_extracted_metadata(func, output) | ||
|
||
return output | ||
|
||
@classmethod | ||
def from_function_pointer(cls, f, autobatch=False, function_kwargs=None): | ||
"""Construct the module given a function pointer | ||
|
||
Args: | ||
f (object): A function pointer | ||
f (object): Function pointer to the Python function to be published | ||
autobatch (bool): Whether to run function on an iterable of entries | ||
function_kwargs (dict): Any default options for this function | ||
""" | ||
return cls.create_model(f.__module__, f.__name__, autobatch, function_kwargs) | ||
return cls.create_model(f=f, autobatch=autobatch, function_kwargs=function_kwargs) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what to do about There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm not sure that I understand.. |
||
|
||
def _get_handler(self): | ||
return 'python.PythonStaticMethodServable' | ||
|
||
def _get_type(self): | ||
return 'Python static method' | ||
|
||
|
||
def add_extracted_metadata(func, model: BasePythonServableModel) -> BasePythonServableModel: | ||
"""Helper function for adding generated input/output metadata to a model object | ||
Args: | ||
func: a pointer to the function whose data is to be extracted | ||
model (BasePythonServableModel): the model who needs its data to be updated | ||
Returns: | ||
(BasePythonServableModel): the model that was given after it is updated | ||
""" | ||
sig = Signature.from_callable(func) | ||
model = model.set_inputs(**signature_to_input(sig)) | ||
model = model.set_outputs(**signature_to_output(sig)) | ||
return model |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
please add more context for the user on what the servable options are -- ideally, link to documentation