Cross Validation Added

gramex/handlers/mlhandler.py

@@ -20,6 +20,10 @@
 from slugify import slugify
 from tornado.gen import coroutine
 from tornado.web import HTTPError
+from sklearn.metrics import get_scorer
+from sklearn.model_selection import cross_val_predict, cross_val_score
+from sklearn.model_selection import cross_val_predict, cross_val_score
+from ast import literal_eval
 
 op = os.path
 MLCLASS_MODULES = [
@@ -40,6 +44,8 @@
     'nums': [],
     'cats': [],
     'target_col': None,
+    'CV': True,
+    'CVargs': []
 }
 ACTIONS = ['predict', 'score', 'append', 'train', 'retrain']
 DEFAULT_TEMPLATE = op.join(op.dirname(__file__), '..', 'apps', 'mlhandler', 'template.html')
@@ -103,7 +109,6 @@ def setup(cls, data=None, model={}, config_dir='', **kwargs):
 
         cls.set_opt('class', model.get('class'))
         cls.set_opt('params', model.get('params', {}))
-
         if op.exists(cls.model_path):  # If the pkl exists, load it
             cls.model = joblib.load(cls.model_path)
         elif data is not None:
@@ -112,14 +117,38 @@ def setup(cls, data=None, model={}, config_dir='', **kwargs):
             data = cls._filtercols(data)
             data = cls._filterrows(data)
             cls.model = cls._assemble_pipeline(data, mclass=mclass, params=params)
-
             # train the model
             target = data[target_col]
             train = data[[c for c in data if c != target_col]]
+            # cross validation
+            print('yayyy we are here')
+            cls.CrossValidation(train,target)
+            print('should have printed')
             gramex.service.threadpool.submit(
                 _fit, cls.model, train, target, cls.model_path, cls.name)
         cls.config_store.flush()
 
+    @classmethod
+    def modelFunction(cls, mclass = ''):
+        model_kwargs = cls.config_store.load('model', {})
+        mclass = model_kwargs.get('class', False)
+        if mclass:
+            model = search_modelclass(mclass)(**model_kwargs.get('params', {}))
+            return model
+
+    @classmethod
+    def CrossValidation(cls,train,target):
+        mod = cls.modelFunction()
+        CV = cls.get_opt('CV') #can edit to make CV true/false etc.
+        if CV:
+            CVargs = cls.get_opt('CVargs')
+            if CVargs:
+                CVscore = cross_val_score(mod, X=train, y=target, **literal_eval(json.dumps(CVargs)))
+            else:
+                CVscore = cross_val_score(mod, train, target)
+            CV = sum(CVscore)/len(CVscore)
+            print('CV score: ', CV)
+
     @classmethod
     def load_data(cls, default=pd.DataFrame()):
         try:
@@ -268,6 +297,10 @@ def _predict(self, data=None, score_col=''):
         self.model = cache.open(self.model_path, joblib.load)
         try:
             target = data.pop(score_col)
+            metric = self.get_argument('_metric', False)
+            if metric:
+                scorer = get_scorer(metric)
+                return scorer(self.model, data, target)
             return self.model.score(data, target)
         except KeyError:
             # Set data in the same order as the transformer requests
@@ -347,6 +380,8 @@ def _train(self, data=None):
         target = data[target_col]
         train = data[[c for c in data if c != target_col]]
         self.model = self._assemble_pipeline(data, force=True)
+        print('IN TRAIN')
+        self.CrossValidation(train,target)
         _fit(self.model, train, target, self.model_path)
         return {'score': self.model.score(train, target)}
 
@@ -357,6 +392,8 @@ def _score(self):
         self._check_model_path()
         data = self._parse_data(False)
         target_col = self.get_argument('target_col', self.get_opt('target_col'))
+        print('IN _SCORE')
+        #self.CrossValidation(data,target_col)
         self.set_opt('target_col', target_col)
         return {'score': self._predict(data, target_col)}
 

tests/test_mlhandler.py

@@ -266,6 +266,11 @@ def test_get_bulk_score(self):
             data=self.df.to_json(orient='records'),
             headers={'Content-Type': 'application/json'})
         self.assertGreaterEqual(resp.json()['score'], self.ACC_TOL)
+        resp = self.get(
+            '/mlhandler?_action=score&_metric=f1_weighted', method='post',
+            data=self.df.to_json(orient='records'),
+            headers={'Content-Type': 'application/json'})
+        self.assertGreaterEqual(resp.json()['score'], self.ACC_TOL)
 
     def test_get_cache(self):
         df = pd.DataFrame.from_records(self.get('/mlhandler?_cache=true').json())