-
Notifications
You must be signed in to change notification settings - Fork 56
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Cross Validation Added #407
base: master
Are you sure you want to change the base?
Changes from 10 commits
cdc0e5a
e737117
64e6c22
babc4bf
b1683b8
ddd5d73
4b672cd
1063207
09b49f1
5f24173
eaa58dc
e175cfe
5f631c1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -20,6 +20,9 @@ | |
from slugify import slugify | ||
from tornado.gen import coroutine | ||
from tornado.web import HTTPError | ||
from sklearn.metrics import get_scorer | ||
from sklearn.model_selection import cross_val_predict, cross_val_score | ||
from ast import literal_eval | ||
|
||
op = os.path | ||
MLCLASS_MODULES = [ | ||
|
@@ -40,6 +43,7 @@ | |
'nums': [], | ||
'cats': [], | ||
'target_col': None, | ||
'CV': True, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Make it lowercase. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We have to support three cases for the
|
||
} | ||
ACTIONS = ['predict', 'score', 'append', 'train', 'retrain'] | ||
DEFAULT_TEMPLATE = op.join(op.dirname(__file__), '..', 'apps', 'mlhandler', 'template.html') | ||
|
@@ -103,7 +107,6 @@ def setup(cls, data=None, model={}, config_dir='', **kwargs): | |
|
||
cls.set_opt('class', model.get('class')) | ||
cls.set_opt('params', model.get('params', {})) | ||
|
||
if op.exists(cls.model_path): # If the pkl exists, load it | ||
cls.model = joblib.load(cls.model_path) | ||
elif data is not None: | ||
|
@@ -112,14 +115,32 @@ def setup(cls, data=None, model={}, config_dir='', **kwargs): | |
data = cls._filtercols(data) | ||
data = cls._filterrows(data) | ||
cls.model = cls._assemble_pipeline(data, mclass=mclass, params=params) | ||
|
||
# train the model | ||
target = data[target_col] | ||
train = data[[c for c in data if c != target_col]] | ||
# cross validation | ||
cls.CrossValidation(train,target) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Make it lowercase. |
||
gramex.service.threadpool.submit( | ||
_fit, cls.model, train, target, cls.model_path, cls.name) | ||
cls.config_store.flush() | ||
|
||
@classmethod | ||
def modelFunction(cls, mclass = ''): | ||
model_kwargs = cls.config_store.load('model', {}) | ||
mclass = model_kwargs.get('class', False) | ||
if mclass: | ||
model = search_modelclass(mclass)(**model_kwargs.get('params', {})) | ||
return model | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This function is not required. |
||
|
||
@classmethod | ||
def CrossValidation(cls,train,target): | ||
mod = cls.modelFunction() | ||
CV = cls.get_opt('CV') | ||
if CV: | ||
CVscore = cross_val_score(mod, X=train, y=target, **literal_eval(json.dumps(CV))) | ||
CVavg = sum(CVscore)/len(CVscore) | ||
print('Cross Validation Score : ',CVavg) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. CV should take place within the train method only. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if cv:
cvscore = cross_val_score(mod, X=train, y=target, cv=cv)
else:
# Do the usual .fit |
||
|
||
@classmethod | ||
def load_data(cls, default=pd.DataFrame()): | ||
try: | ||
|
@@ -268,6 +289,10 @@ def _predict(self, data=None, score_col=''): | |
self.model = cache.open(self.model_path, joblib.load) | ||
try: | ||
target = data.pop(score_col) | ||
metric = self.get_argument('_metric', False) | ||
if metric: | ||
scorer = get_scorer(metric) | ||
return scorer(self.model, data, target) | ||
return self.model.score(data, target) | ||
except KeyError: | ||
# Set data in the same order as the transformer requests | ||
|
@@ -347,6 +372,7 @@ def _train(self, data=None): | |
target = data[target_col] | ||
train = data[[c for c in data if c != target_col]] | ||
self.model = self._assemble_pipeline(data, force=True) | ||
self.CrossValidation(train,target) | ||
_fit(self.model, train, target, self.model_path) | ||
return {'score': self.model.score(train, target)} | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This should not be required.