Source code for autogluon.tabular.models.lr.lr_model
import logging
import re
import numpy as np
from pandas import DataFrame
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.impute import SimpleImputer
from sklearn.pipeline import FeatureUnion, Pipeline
from sklearn.preprocessing import StandardScaler, QuantileTransformer
from autogluon.core.constants import BINARY, REGRESSION
from autogluon.core.features.types import R_INT, R_FLOAT, R_CATEGORY, R_OBJECT
from .hyperparameters.parameters import get_param_baseline, INCLUDE, IGNORE, ONLY, _get_solver, preprocess_params_set
from .hyperparameters.searchspaces import get_default_searchspace
from .lr_preprocessing_utils import NlpDataPreprocessor, OheFeaturesGenerator, NumericDataPreprocessor
from autogluon.core.models.abstract.model_trial import skip_hpo
from autogluon.core.models import AbstractModel
logger = logging.getLogger(__name__)
# TODO: Can Bagged LinearModels be combined during inference to 1 model by averaging their weights?
# What about just always using refit_full model? Should we even bag at all? Do we care that its slightly overfit?
[docs]class LinearModel(AbstractModel):
"""
Linear model (scikit-learn): https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
Model backend differs depending on problem_type:
'binary' & 'multiclass': https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
'regression': https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._pipeline = None
def _get_model_type(self):
penalty = self.params.get('penalty', 'L2')
if self.params_aux.get('use_daal', False):
# Disabled by default until more testing is done, appears to give 20x training speedup when enabled
try:
# TODO: Add more granular switch, currently this affects all future LR models even if they had `use_daal=False`
from sklearnex import patch_sklearn
patch_sklearn("ridge")
patch_sklearn("lasso")
patch_sklearn("logistic")
logger.log(15, '\tUsing daal4py LR backend...')
except:
pass
from sklearn.linear_model import LogisticRegression, Ridge, Lasso
if self.problem_type == REGRESSION:
if penalty == 'L2':
model_type = Ridge
elif penalty == 'L1':
model_type = Lasso
else:
raise AssertionError(f'Unknown value for penalty "{penalty}" - supported types are ["L1", "L2"]')
else:
model_type = LogisticRegression
return model_type
def _tokenize(self, s):
return re.split('[ ]+', s)
def _get_types_of_features(self, df):
""" Returns dict with keys: : 'continuous', 'skewed', 'onehot', 'embed', 'language', values = ordered list of feature-names falling into each category.
Each value is a list of feature-names corresponding to columns in original dataframe.
TODO: ensure features with zero variance have already been removed before this function is called.
"""
feature_types = self._feature_metadata.get_type_group_map_raw()
categorical_featnames = feature_types[R_CATEGORY] + feature_types[R_OBJECT] + feature_types['bool']
continuous_featnames = feature_types[R_FLOAT] + feature_types[R_INT] # + self.__get_feature_type_if_present('datetime')
language_featnames = [] # TODO: Disabled currently, have to pass raw text data features here to function properly
valid_features = categorical_featnames + continuous_featnames + language_featnames
if len(categorical_featnames) + len(continuous_featnames) + len(language_featnames) != df.shape[1]:
unknown_features = [feature for feature in df.columns if feature not in valid_features]
df = df.drop(columns=unknown_features)
self._features_internal = list(df.columns) # FIXME: Don't edit _features_internal
types_of_features = {'continuous': [], 'skewed': [], 'onehot': [], 'language': []}
return self._select_features(df, types_of_features, categorical_featnames, language_featnames, continuous_featnames)
def _select_features(self, df, types_of_features, categorical_featnames, language_featnames, continuous_featnames):
features_selector = {
INCLUDE: self._select_features_handle_text_include,
ONLY: self._select_features_handle_text_only,
IGNORE: self._select_features_handle_text_ignore,
}.get(self.params.get('handle_text', IGNORE), self._select_features_handle_text_ignore)
return features_selector(df, types_of_features, categorical_featnames, language_featnames, continuous_featnames)
# TODO: handle collinear features - they will impact results quality
def _preprocess(self, X, is_train=False, **kwargs):
if is_train:
feature_types = self._get_types_of_features(X)
X = self._preprocess_train(X, feature_types, self.params['vectorizer_dict_size'])
else:
X = self._pipeline.transform(X)
return X
def _preprocess_train(self, X, feature_types, vect_max_features):
transformer_list = []
if len(feature_types['language']) > 0:
pipeline = Pipeline(steps=[
("preparator", NlpDataPreprocessor(nlp_cols=feature_types['language'])),
("vectorizer", TfidfVectorizer(ngram_range=self.params['proc.ngram_range'], sublinear_tf=True, max_features=vect_max_features, tokenizer=self._tokenize)),
])
transformer_list.append(('vect', pipeline))
if len(feature_types['onehot']) > 0:
pipeline = Pipeline(steps=[
('generator', OheFeaturesGenerator(cats_cols=feature_types['onehot'])),
])
transformer_list.append(('cats', pipeline))
if len(feature_types['continuous']) > 0:
pipeline = Pipeline(steps=[
('generator', NumericDataPreprocessor(cont_cols=feature_types['continuous'])),
('imputer', SimpleImputer(strategy=self.params['proc.impute_strategy'])),
('scaler', StandardScaler())
])
transformer_list.append(('cont', pipeline))
if len(feature_types['skewed']) > 0:
pipeline = Pipeline(steps=[
('generator', NumericDataPreprocessor(cont_cols=feature_types['skewed'])),
('imputer', SimpleImputer(strategy=self.params['proc.impute_strategy'])),
('quantile', QuantileTransformer(output_distribution='normal')), # Or output_distribution = 'uniform'
])
transformer_list.append(('skew', pipeline))
self._pipeline = FeatureUnion(transformer_list=transformer_list)
return self._pipeline.fit_transform(X)
def _set_default_params(self):
default_params = {'random_state': 0, 'fit_intercept': True}
if self.problem_type != REGRESSION:
default_params.update({'solver': _get_solver(self.problem_type), 'n_jobs': -1})
default_params.update(get_param_baseline())
for param, val in default_params.items():
self._set_default_param_value(param, val)
def _get_default_searchspace(self):
return get_default_searchspace(self.problem_type)
# TODO: It could be possible to adaptively set max_iter [1] to approximately respect time_limit based on sample-size, feature-dimensionality, and the solver used.
# [1] https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#examples-using-sklearn-linear-model-logisticregression
def _fit(self,
X,
y,
sample_weight=None,
**kwargs):
X = self.preprocess(X, is_train=True)
if self.problem_type == BINARY:
y = y.astype(int).values
params = {k: v for k, v in self.params.items() if k not in preprocess_params_set}
# Ridge/Lasso are using alpha instead of C, which is C^-1
# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge
if self.problem_type == REGRESSION and 'alpha' not in params:
# For numerical reasons, using alpha = 0 with the Lasso object is not advised, so we add epsilon
params['alpha'] = 1 / (params['C'] if params['C'] != 0 else 1e-8)
params.pop('C', None)
# TODO: copy_X=True currently set during regression problem type, could potentially set to False to avoid unnecessary data copy.
model_cls = self._get_model_type()
model = model_cls(**params)
logger.log(15, f'Training Model with the following hyperparameter settings:')
logger.log(15, model)
if sample_weight is None:
self.model = model.fit(X, y) # Necessary for rapids models
else:
self.model = model.fit(X, y, sample_weight=sample_weight)
# TODO: Add HPO
def _hyperparameter_tune(self, **kwargs):
return skip_hpo(self, **kwargs)
def _select_features_handle_text_include(self, df, types_of_features, categorical_featnames, language_featnames, continuous_featnames):
# continuous = numeric features to rescale
# skewed = features to which we will apply power (ie. log / box-cox) transform before normalization
# onehot = features to one-hot encode (unknown categories for these features encountered at test-time are encoded as all zeros). We one-hot encode any features encountered that only have two unique values.
one_hot_threshold = 10000 # FIXME research memory constraints
for feature in self._features_internal:
feature_data = df[feature]
num_unique_vals = len(feature_data.unique())
if feature in language_featnames:
types_of_features['language'].append(feature)
elif feature in continuous_featnames:
if np.abs(feature_data.skew()) > self.params['proc.skew_threshold']:
types_of_features['skewed'].append(feature)
else:
types_of_features['continuous'].append(feature)
elif (feature in categorical_featnames) and (num_unique_vals <= one_hot_threshold):
types_of_features['onehot'].append(feature)
return types_of_features
def _select_features_handle_text_only(self, df, types_of_features, categorical_featnames, language_featnames, continuous_featnames):
for feature in self._features_internal:
if feature in language_featnames:
types_of_features['language'].append(feature)
return types_of_features
def _select_features_handle_text_ignore(self, df, types_of_features, categorical_featnames, language_featnames, continuous_featnames):
# continuous = numeric features to rescale
# skewed = features to which we will apply power (ie. log / box-cox) transform before normalization
# onehot = features to one-hot encode (unknown categories for these features encountered at test-time are encoded as all zeros). We one-hot encode any features encountered that only have two unique values.
one_hot_threshold = 10000 # FIXME research memory constraints
for feature in self._features_internal:
feature_data = df[feature]
num_unique_vals = len(feature_data.unique())
if feature in continuous_featnames:
if np.abs(feature_data.skew()) > self.params['proc.skew_threshold']:
types_of_features['skewed'].append(feature)
else:
types_of_features['continuous'].append(feature)
elif (feature in categorical_featnames) and (num_unique_vals <= one_hot_threshold):
types_of_features['onehot'].append(feature)
return types_of_features
def _get_default_auxiliary_params(self) -> dict:
default_auxiliary_params = super()._get_default_auxiliary_params()
extra_auxiliary_params = dict(
ignored_type_group_raw=[R_OBJECT],
)
default_auxiliary_params.update(extra_auxiliary_params)
return default_auxiliary_params