TextPredictor

.github/workflows/python-package.yml

@@ -47,6 +47,10 @@ jobs:
         run: |
           pip install -e .[ray,forecast]
           pip install 'tensorboardX<=2.2'
+      - name: If python version > 3.6 and not on windows, install autogluon
+        if: matrix.python-version >= '3.7' && (matrix.os == 'macOS-latest' || matrix.os == 'ubuntu-latest')
+        run: |
+          pip install -e .[autogluon]
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names

flaml/data.py

@@ -272,7 +272,8 @@ def fit_transform(self, X: Union[DataFrame, np.array], y, task):
         elif isinstance(X, DataFrame):
             X = X.copy()
             n = X.shape[0]
-            cat_columns, num_columns, datetime_columns = [], [], []
+            # NOTE: add str_columns here
+            str_columns, cat_columns, num_columns, datetime_columns = [], [], [], []
             drop = False
             if task in TS_FORECAST:
                 X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})
@@ -282,13 +283,17 @@ def fit_transform(self, X: Union[DataFrame, np.array], y, task):
             for column in X.columns:
                 # sklearn\utils\validation.py needs int/float values
                 if X[column].dtype.name in ("object", "category"):
-                    if (
-                        X[column].nunique() == 1
-                        or X[column].nunique(dropna=True)
-                        == n - X[column].isnull().sum()
-                    ):
+                    if X[column].nunique() == 1:
                         X.drop(columns=column, inplace=True)
                         drop = True
+                    elif X[column].nunique(dropna=True) >= int((n - X[column].isnull().sum()) * 0.9):
+                    # NOTE: here a threshold is applied for distinguishing str vs. cat 
+                    # if no threshold wanted => requires every non-nan str entry to be different
+                    # delete the line above and uncomment below
+                    # elif X[column].nunique(dropna=True) == n - X[column].isnull().sum():
+                        # NOTE: here detects str fields and do fillna with ""
+                        X[column] = X[column].fillna("")
+                        str_columns.append(column)
                     elif X[column].dtype.name == "category":
                         current_categories = X[column].cat.categories
                         if "__NAN__" not in current_categories:
@@ -330,7 +335,7 @@ def fit_transform(self, X: Union[DataFrame, np.array], y, task):
                         del tmp_dt
                     X[column] = X[column].fillna(np.nan)
                     num_columns.append(column)
-            X = X[cat_columns + num_columns]
+            X = X[str_columns + cat_columns + num_columns]
             if task in TS_FORECAST:
                 X.insert(0, TS_TIMESTAMP_COL, ds_col)
             if cat_columns:
@@ -359,7 +364,8 @@ def fit_transform(self, X: Union[DataFrame, np.array], y, task):
                     ]
                 )
                 X[num_columns] = self.transformer.fit_transform(X_num)
-            self._cat_columns, self._num_columns, self._datetime_columns = (
+            self.str_columns, self._cat_columns, self._num_columns, self._datetime_columns = (
+                str_columns,
                 cat_columns,
                 num_columns,
                 datetime_columns,
@@ -400,7 +406,8 @@ def transform(self, X: Union[DataFrame, np.array]):
             if len(self._str_columns) > 0:
                 X[self._str_columns] = X[self._str_columns].astype("string")
         elif isinstance(X, DataFrame):
-            cat_columns, num_columns, datetime_columns = (
+            str_columns, cat_columns, num_columns, datetime_columns = (
+                self.str_columns,
                 self._cat_columns,
                 self._num_columns,
                 self._datetime_columns,
@@ -426,7 +433,7 @@ def transform(self, X: Union[DataFrame, np.array]):
                         X[new_col_name] = new_col_value
                 X[column] = X[column].map(datetime.toordinal)
                 del tmp_dt
-            X = X[cat_columns + num_columns].copy()
+            X = X[str_columns + cat_columns + num_columns].copy()
             if self._task in TS_FORECAST:
                 X.insert(0, TS_TIMESTAMP_COL, ds_col)
             for column in cat_columns:

flaml/ml.py

@@ -37,6 +37,7 @@
     ARIMA,
     SARIMAX,
     TransformersEstimator,
+    MultiModalEstimator,
 )
 from .data import CLASSIFICATION, group_counts, TS_FORECAST, TS_VALUE_COL
 import logging
@@ -121,6 +122,8 @@ def get_estimator_class(task, estimator_name):
         estimator_class = SARIMAX
     elif estimator_name == "transformer":
         estimator_class = TransformersEstimator
+    elif estimator_name == "multimodal":
+        estimator_class = MultiModalEstimator
     else:
         raise ValueError(
             estimator_name + " is not a built-in learner. "
@@ -575,6 +578,9 @@ def compute_estimator(
         fit_kwargs["X_val"] = X_val
         fit_kwargs["y_val"] = y_val
 
+    elif isinstance(estimator, MultiModalEstimator):
+        fit_kwargs["metric"] = eval_metric
+
     if "holdout" == eval_method:
         val_loss, metric_for_logging, train_time, pred_time = get_val_loss(
             config_dic,

flaml/model.py

@@ -2099,6 +2099,129 @@ class XGBoostLimitDepth_TS(TS_SKLearn):
     base_class = XGBoostLimitDepthEstimator
 
 
+class MultiModalEstimator(BaseEstimator):
+    """
+    The class for tuning AutoGluon TextPredictor
+    """
+    @classmethod
+    def search_space(cls, **params):
+        """
+        Add the possible search space configs here, e.g. 'optimization.lr'
+        reference:
+        https://auto.gluon.ai/stable/tutorials/text_prediction/customization.html#custom-hyperparameter-values
+        """
+        # TODO: expand the search space
+        search_space_dict = {
+            "model.network.agg_net.mid_units": {
+                "domain": tune.choice(list(range(32, 129))),
+                "init_value": 128,
+            },
+            "optimization.lr": {
+                "domain": tune.loguniform(lower=1E-5, upper=1E-4),
+                "init_value": 1E-4,
+            },
+            "optimization.wd": {
+                "domain": tune.choice([1E-4, 1E-3, 1E-2]),
+                "init_value": 1E-4,
+            },
+            "optimization.warmup_portion": {
+                "domain": tune.choice([0.1, 0.2]),
+                "init_value": 0.1, 
+            },
+            "optimization.layerwise_lr_decay": {
+                "domain": tune.choice([0.8, 0.9]),
+                "init_value": 0.8,
+            },
+            "optimization.nbest": {
+                "domain": tune.choice([2, 3, 4,]),
+                "init_value": 3,
+            },
+            "optimization.num_train_epochs": {
+                "domain": tune.choice([5, 10, 15,]),
+                "init_value": 10,
+            },
+            "optimization.per_device_batch_size": {
+                "domain": tune.choice([2, 4, 8,]),
+                "init_value": 10,
+            },
+            "optimization.batch_size": {
+                "domain": tune.choice([32, 64, 128,]),
+                "init_value": 128,
+            },
+        }
+        return search_space_dict
+
+    def _init_ag_args(self, automl_fit_kwargs: dict = None):
+        from .nlp.utils import AGArgs
+
+        ag_args = AGArgs()
+        for key, val in automl_fit_kwargs["ag_args"].items():
+            assert (
+                key in ag_args.__dict__
+            ), "The specified key {} is not in the argument list of flaml.nlp.utils::AGArgs".format(
+                key
+            )
+            setattr(ag_args, key, val)
+        self.ag_args = ag_args
+
+    def _set_seed(self, seed):
+        import random
+        import mxnet as mx
+        # NOTE: if support pytorch backend, uncomment below
+        # import torch as th
+        # th.manual_seed(seed)
+        mx.random.seed(seed)
+        np.random.seed(seed)
+        random.seed(seed)
+
+    def fit(self, X_train=None, y_train=None, budget=None, **kwargs):
+        from autogluon.text import TextPredictor
+
+        self._kwargs = kwargs
+        self._init_ag_args(kwargs)
+        seed = self._kwargs.get("seed", 123)
+        self._set_seed(seed)
+
+        assert (self.ag_args.backend == "mxnet"), "the pytorch automm model is not supported. "
+        # get & set the hyperparameters, update with self.params
+        hyperparameters = self.ag_args.get_presets()
+        search_space = hyperparameters["models"]["MultimodalTextModel"]["search_space"]
+        for key, value in self.params.items():
+            # NOTE: FLAML uses np.float64 but AG uses float, need to transform
+            if key == "n_jobs": 
+                continue
+            elif isinstance(value, np.float64):
+                search_space[key] = value.item()
+            else:
+                search_space[key] = value
+        start_time = time.time()
+        self._model = TextPredictor(path=self.ag_args.output_dir,
+                                    label="label",
+                                    problem_type=self._task,
+                                    eval_metric=kwargs["metric"],
+                                    backend=self.ag_args.backend)
+        train_data = TransformersEstimator._join(X_train, y_train)
+        self._model.fit(train_data=train_data,
+                        hyperparameters=hyperparameters,
+                        time_limit=budget,
+                        seed=seed)
+
+        training_time = time.time() - start_time
+        return training_time
+
+    def predict(self, X):
+        output = self._model.predict(X, as_pandas=False)
+        return output
+
+    def predict_proba(self, X):
+        # only works for classification tasks
+        assert (
+            self._task in CLASSIFICATION
+        ), "predict_proba() only for classification tasks."
+        output = self._model.predict_proba(X, as_pandas=False)
+        return output
+
+
 class suppress_stdout_stderr(object):
     def __init__(self):
         # Open a pair of null files

flaml/nlp/utils.py

@@ -632,3 +632,84 @@ def load_args_from_console():
             )
         console_args, unknown = arg_parser.parse_known_args()
         return console_args
+
+
+@dataclass
+class AGArgs:
+    """
+    The Autogluon configurations
+    Args:
+        output_dir (str): data root directory for outputing the log and intermediate data, model.
+        backend (str, optional, defaults to "mxnet"): currently only support to mxnet.
+        text_backbone (str, optional, defaults to "electra_base"): the text backbone model.
+        multimodal_fusion_strategy (str, optional, defaults to "fuse_late"): the fuse strategy.
+    """
+    output_dir: str = field(
+        default="data/mm/output/", metadata={"help": "data dir", "required": True}
+    )
+    backend: str = field(default="mxnet", metadata={"help": "the backend of the multimodal model"})
+    text_backbone: str = field(default="electra_base", metadata={"help": "text backbone model"})
+    multimodal_fusion_strategy: str = field(default="fuse_late", metadata={"help": "fusion strategy"})
+    # TODO: determine whether to tune these HPs 
+    # per_device_batch_size: int = field(default=8, metadata={"help": "per device batch size"})
+    # num_train_epochs: int = field(default=10, metadata={"help": "number of train epochs"})
+    # batch_size: int = field(default=128,  metadata={"help": "batch size"})
+
+
+    def get_presets(self):
+        """
+        Get the preset using the AGArgs.
+        {'models': {'MultimodalTextModel': {'backend': 'gluonnlp_v0',
+                                    'search_space': {'model.backbone.name': 'google_electra_small',
+                                                    'model.network.agg_net.agg_type': 'concat',
+                                                    'model.network.agg_net.mid_units': 128,  # [in HPO example]
+                                                    'model.network.aggregate_categorical': True,
+                                                    'model.use_avg_nbest': True,
+                                                    'optimization.batch_size': 128,
+                                                    'optimization.layerwise_lr_decay': 0.8,
+                                                    'optimization.lr': Categorical[0.0001],
+                                                    'optimization.nbest': 3,
+                                                    'optimization.num_train_epochs': 10,
+                                                    'optimization.per_device_batch_size': 8,
+                                                    'optimization.wd': 0.0001,
+                                                    'optimization.warmup_portion': 0.1,  # [in HPO example]
+                                                    'preprocessing.categorical.convert_to_text': False,
+                                                    'preprocessing.numerical.convert_to_text': False}}},
+        'tune_kwargs': {'num_trials': 1,
+                        'scheduler_options': None,
+                        'search_options': None,
+                        'search_strategy': 'local',
+                        'searcher': 'random'}}
+        Ref: https://auto.gluon.ai/0.3.1/tutorials/text_prediction/customization.html
+        Return:
+            hyperparameters: a Dict of the preset hyperparameter settings.
+        """
+        from autogluon.text.text_prediction.legacy_presets import ag_text_presets
+
+        base_key = f'{self.text_backbone}_{self.multimodal_fusion_strategy}'
+        hyperparameters = ag_text_presets.create(base_key)
+        # NOTE: set anything else that would like to be set via ag_args here
+        return hyperparameters
+
+
+    @staticmethod
+    def load_args():
+        from dataclasses import fields
+
+        arg_parser = argparse.ArgumentParser()
+        for each_field in fields(AGArgs):
+            print(each_field)
+            arg_parser.add_argument(
+                "--" + each_field.name,
+                type=each_field.type,
+                help=each_field.metadata["help"],
+                required=each_field.metadata["required"]
+                if "required" in each_field.metadata
+                else False,
+                choices=each_field.metadata["choices"]
+                if "choices" in each_field.metadata
+                else None,
+                default=each_field.default,
+            )
+        console_args, unknown = arg_parser.parse_known_args()
+        return console_args

setup.py

@@ -63,6 +63,11 @@
             "hcrystalball==0.1.10",
             "seqeval",
         ],
+        "autogluon": [
+            "mxnet<2.0.0",
+            "autogluon.text==0.4.0",
+            "autogluon.features==0.4.0",
+        ],
         "catboost": ["catboost>=0.26"],
         "blendsearch": ["optuna==2.8.0"],
         "ray": [