TextPredictor

.github/workflows/python-package.yml

@@ -47,6 +47,10 @@ jobs:
         run: |
           pip install -e .[ray,forecast]
           pip install 'tensorboardX<=2.2'
+      - name: If python version > 3.6 and not on windows, install autogluon
+        if: matrix.python-version >= '3.7' && (matrix.os == 'macOS-latest' || matrix.os == 'ubuntu-latest')
+        run: |
+          pip install -e .[autogluon]
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names

flaml/automl.py

@@ -46,6 +46,7 @@
     REGRESSION,
     _is_nlp_task,
     NLG_TASKS,
+    _is_mm_task,
 )
 from . import tune
 from .training_log import training_log_reader, training_log_writer
@@ -974,7 +975,8 @@ def _validate_data(
                 "or all columns of X are integer ids (tokenized)"
             )
 
-        if issparse(X_train_all):
+        if issparse(X_train_all) or _is_mm_task(self._state.task):
+            # leave the preprocessing to the mm_estimator
             self._transformer = self._label_transformer = False
             self._X_train_all, self._y_train_all = X, y
         else:

flaml/data.py

@@ -23,9 +23,11 @@
     SEQCLASSIFICATION,
     MULTICHOICECLASSIFICATION,
     TOKENCLASSIFICATION,
+    "mm_multi",
+    "mm_binary",
 )
 SEQREGRESSION = "seq-regression"
-REGRESSION = ("regression", SEQREGRESSION)
+REGRESSION = ("regression", "mm_regression", SEQREGRESSION)
 TS_FORECASTREGRESSION = (
     "forecast",
     "ts_forecast",
@@ -47,6 +49,14 @@
     TOKENCLASSIFICATION,
 )
 
+MM_TASKS = ("mm_binary", "mm_multi", "mm_regression")
+
+
+ ## ***** ADDED FOR MULTIMODAL *****
+def _is_mm_task(task):
+    return True if task in MM_TASKS else False
+## ***** END ADDED FOR MULTIMODAL *****
+
 
 def _is_nlp_task(task):
     if task in NLU_TASKS or task in NLG_TASKS:

flaml/ml.py

@@ -37,6 +37,7 @@
     ARIMA,
     SARIMAX,
     TransformersEstimator,
+    AGTextPredictorEstimator,
 )
 from .data import CLASSIFICATION, group_counts, TS_FORECAST, TS_VALUE_COL
 import logging
@@ -121,6 +122,8 @@ def get_estimator_class(task, estimator_name):
         estimator_class = SARIMAX
     elif estimator_name == "transformer":
         estimator_class = TransformersEstimator
+    elif estimator_name == "agtextpredictor":
+        estimator_class = AGTextPredictorEstimator
     else:
         raise ValueError(
             estimator_name + " is not a built-in learner. "

flaml/model.py

@@ -1965,6 +1965,189 @@ class XGBoostLimitDepth_TS(TS_SKLearn):
     base_class = XGBoostLimitDepthEstimator
 
 
+class AGTextPredictorEstimator(BaseEstimator):
+    """
+    The class for tuning AutoGluon TextPredictor
+    """
+    def __init__(self, task="binary", **params,):
+        from autogluon.text import TextPredictor
+
+        super().__init__(task, **params)
+        self.estimator_class = TextPredictor
+
+    @classmethod
+    def search_space(cls, **params):
+        """
+        Add the possible search space configs here, e.g. 'optimization.lr'
+        reference:
+        https://auto.gluon.ai/stable/tutorials/text_prediction/customization.html#custom-hyperparameter-values
+        """
+        search_space_dict = {
+            "model.network.agg_net.mid_units": {
+                "domain": tune.choice(list(range(32, 129))),
+                "init_value": 128,
+            },
+            "optimization.lr": {
+                "domain": tune.loguniform(lower=1E-5, upper=1E-4),
+                "init_value": 1E-4,
+            },
+            "optimization.wd": {
+                "domain": tune.choice([1E-4, 1E-3, 1E-2]),
+                "init_value":1E-4,
+            },
+            "optimization.warmup_portion": {
+                "domain": tune.choice([0.1, 0.2]),
+                "init_value":0.1, 
+            },
+        }
+        return search_space_dict
+
+    def _init_fix_args(self, automl_fit_kwargs: dict=None):
+        """
+        Save the customed fix args here
+        this includes:
+            "output_dir",
+            "text_backbone": "electra_base"
+            "multimodal_fusion_strategy":"fuse_late",
+        """
+        fix_args = {}
+        FIX_ARGS_LIST = ["output_dir", "dataset_name", "label_column", "per_device_batch_size", "backend",
+                         "text_backbone", "multimodal_fusion_strategy", "num_train_epochs", "batch_size",]
+        for key, value in automl_fit_kwargs["custom_fix_args"].items():
+            assert (
+                key in FIX_ARGS_LIST
+            ), "The specified key {} is not in the argument list: output_dir, backend, label_column, dataset_name, text_backbone,\
+                multimodal_fusion_strategy, num_train_epochs, batch_size, per_device_batch_size".format(key)
+
+            fix_args[key] = value
+
+        self.fix_args = fix_args
+
+    def _init_hp_config(self, text_backbone: str, multimodal_fusion_strategy: str):
+        """"
+        Ref:
+        https://auto.gluon.ai/stable/tutorials/text_prediction/customization.html#custom-hyperparameter-values
+        """
+        if self.fix_args.get("backend", "pytorch") == "mxnet":
+            from autogluon.text.text_prediction.legacy_presets import ag_text_presets
+
+            base_key = f'{text_backbone}_{multimodal_fusion_strategy}'
+            cfg = ag_text_presets.create(base_key)
+            # NOTE: if the search_space() is modified, add new items or delete here too.
+            TUNABLE_HP = set(["model.network.agg_net.mid_units",
+                            "optimization.batch_size",
+                            "optimization.layerwise_lr_decay",
+                            "optimization.lr",
+                            "optimization.nbest",
+                            "optimization.num_train_epochs",
+                            "optimization.per_device_batch_size",
+                            "optimization.wd",
+                            "optimization.warmup_portion",
+                            ])
+            search_space = cfg["models"]["MultimodalTextModel"]["search_space"]
+            search_space["optimization.per_device_batch_size"] = self.fix_args.get("per_device_batch_size", 4)
+            search_space["optimization.num_train_epochs"] = self.fix_args.get("num_train_epochs", 10)
+            search_space["optimization.batch_size"] = self.fix_args.get("batch_size", 128)
+            for key, value in self.params.items():
+                if key in TUNABLE_HP:
+                    # NOTE: FLAML uses np.float64 but AG uses float, need to transform
+                    if isinstance(value, np.float64):
+                        search_space[key] = value.item()
+                    else:
+                        search_space[key] = value
+            return cfg
+
+        else:
+            raise ValueError("the pytorch automm model is not supported. ")
+            # from autogluon.text.text_prediction.presets import get_text_preset
+
+            # cfg, overrides = get_text_preset("default")  # get preset for text+num+cat+fusion
+            # # TODO: set the search space for the auto_mm in AG 0.4.0
+            # cfg.hf_text.checkpoint_name = self.fix_args["hf_text.checkpoint_name"]
+            # # get search configs from self.params and set here
+            # TUNABLE_HP = []
+            # for key, value in self.params.items():
+            #     if key in TUNABLE_HP:
+            #         # NOTE: FLAML uses np.float64 but AG uses float, might need to transform
+            #         if isinstance(value, np.float64):
+            #             search_space[key] = value.item()
+            #         else:
+            #             search_space[key] = value
+            return cfg
+
+
+    def _set_seed(self, seed):
+        import random
+        import mxnet as mx
+        import torch as th
+        th.manual_seed(seed)
+        mx.random.seed(seed)
+        np.random.seed(seed)
+        random.seed(seed)
+
+    def fit(self, X_train=None, y_train=None, budget=None, **kwargs):
+        self._kwargs = kwargs
+        self._init_fix_args(kwargs)
+        # the seed set in the bash script for ag experiment is 123
+        seed = self.params.get("seed", 123)
+        self._set_seed(seed)
+
+        # get backbone and fusion strategy
+        text_backbone = self.fix_args["text_backbone"]
+        multimodal_fusion_strategy = self.fix_args["multimodal_fusion_strategy"]
+
+        # get & set the save dir, get the dataset info
+        save_dir = self.fix_args["output_dir"]
+        label_column = self.fix_args["label_column"]
+        dataset_name = self.fix_args["dataset_name"]
+        ag_model_save_dir = os.path.join(save_dir, f"{dataset_name}_ag_text_multimodal_{text_backbone}\
+                                                    _{multimodal_fusion_strategy}_no_ensemble")
+
+        # set the hyperparameters
+        self.hyperparameters = self._init_hp_config(text_backbone, multimodal_fusion_strategy)
+        PROBLEM_TYPE_MAPPING = {"mm_binary": "binary", "mm_multi": "multiclass", "mm_regression": "regression"}
+        TASK_METRIC_MAPPING = {"mm_multi": "acc", "mm_binary": "roc_auc", "mm_regression": "r2"}
+
+        # train the model
+        start_time = time.time()
+
+        self._model = self.estimator_class(path=ag_model_save_dir,
+                                           label=label_column,
+                                           problem_type=PROBLEM_TYPE_MAPPING[self._task],
+                                           eval_metric=TASK_METRIC_MAPPING[self._task],
+                                           backend=self.fix_args.get("backend", "pytorch"))
+
+        # train_data = self._kwargs["train_data"]
+        import pandas as pd
+        train_data = pd.concat([X_train, y_train], axis=1)
+        tuning_data = pd.concat([X_train, y_train], axis=1)
+
+        self._model.fit(train_data=train_data,
+                        tuning_data=kwargs.get("tuning_data", None),
+                        hyperparameters=self.hyperparameters,
+                        time_limit=budget,
+                        seed=seed)
+
+        training_time = time.time() - start_time
+        return training_time
+
+    def predict(self, X):
+        output = self._model.predict(X, as_pandas=False)
+        return output
+
+    def predict_proba(self, X, as_multiclass=True):
+        # only works for classification tasks
+        assert (
+            self._task in CLASSIFICATION
+        ), "predict_proba() only for classification tasks."
+
+        output = self._model.predict_proba(X, as_pandas=False)
+        if not as_multiclass:
+            if self._task == "mm_binary":
+                output = output[:, 1]
+        return output
+
+
 class suppress_stdout_stderr(object):
     def __init__(self):
         # Open a pair of null files

setup.py

@@ -63,6 +63,11 @@
             "hcrystalball==0.1.10",
             "seqeval",
         ],
+        "autogluon": [
+            "mxnet<2.0.0",
+            "autogluon.text==0.4.0",
+            "autogluon.features==0.4.0",
+        ],
         "catboost": ["catboost>=0.26"],
         "blendsearch": ["optuna==2.8.0"],
         "ray": [