Source code for autogluon.tabular.models.rf.rf_model

import logging
import math
import pickle
import sys
import time

import numpy as np
import psutil

from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS, QUANTILE
from autogluon.core.utils.exceptions import NotEnoughMemoryError, TimeLimitExceeded
from autogluon.core.features.types import R_OBJECT
from autogluon.core.utils.utils import normalize_pred_probas

from autogluon.core.models.abstract.model_trial import skip_hpo
from autogluon.core.models import AbstractModel
from autogluon.features.generators import LabelEncoderFeatureGenerator

logger = logging.getLogger(__name__)

[docs]class RFModel(AbstractModel): """ Random Forest model (scikit-learn): """ def __init__(self, **kwargs): super().__init__(**kwargs) self._feature_generator = None self._daal = False # Whether daal4py backend is being used def _get_model_type(self): if self.problem_type == QUANTILE: from .rf_quantile import RandomForestQuantileRegressor return RandomForestQuantileRegressor if self.params_aux.get('use_daal', False): # Disabled by default because it appears to degrade performance try: # TODO: Use sklearnex instead once a suitable toggle option is provided that won't impact future models # FIXME: DAAL OOB score is broken, returns biased predictions. Without this optimization, can't compute Efficient OOF. from daal4py.sklearn.ensemble import RandomForestClassifier, RandomForestRegressor logger.log(15, '\tUsing daal4py RF backend...') self._daal = True except: from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor self._daal = False else: from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor self._daal = False if self.problem_type in [REGRESSION, SOFTCLASS]: return RandomForestRegressor else: return RandomForestClassifier # TODO: X.fillna -inf? Add extra is_missing column? def _preprocess(self, X, **kwargs): X = super()._preprocess(X, **kwargs) if self._feature_generator is None: self._feature_generator = LabelEncoderFeatureGenerator(verbosity=0) if self._feature_generator.features_in: X = X.copy() X[self._feature_generator.features_in] = self._feature_generator.transform(X=X) X = X.fillna(0).to_numpy(dtype=np.float32) return X def _set_default_params(self): default_params = { # TODO: 600 is much better, but increases info leakage in stacking -> therefore 300 is ~equal in stack ensemble final quality. # Consider adding targeted noise to OOF to avoid info leakage, or increase `min_samples_leaf`. 'n_estimators': 300, 'n_jobs': -1, 'random_state': 0, 'bootstrap': True, # Required for OOB estimates, setting to False will raise exception if bagging. # TODO: min_samples_leaf=5 is too large on most problems, however on some datasets it helps a lot (airlines likes >40 min_samples_leaf, adult likes 2 much better than 1) # This value would need to be tuned per dataset, likely very worthwhile. # Higher values = less OOF info leak, default = 1, which maximizes info leak. # 'min_samples_leaf': 5, # Significantly reduces info leakage to stacker models. Never use the default/1 when using as base model. # 'oob_score': True, # Disabled by default as it is better to do it post-fit via custom logic. } for param, val in default_params.items(): self._set_default_param_value(param, val) # TODO: Add in documentation that Categorical default is the first index # TODO: enable HPO for RF models def _get_default_searchspace(self): spaces = { # 'n_estimators': Int(lower=10, upper=1000, default=300), # 'max_features': Categorical(['auto', 0.5, 0.25]), # 'criterion': Categorical(['gini', 'entropy']), } return spaces def _fit(self, X, y, time_limit=None, sample_weight=None, **kwargs): time_start = time.time() model_cls = self._get_model_type() max_memory_usage_ratio = self.params_aux['max_memory_usage_ratio'] params = self._get_model_params() n_estimators_final = params['n_estimators'] n_estimators_minimum = min(40, n_estimators_final) n_estimators_test = min(4, max(1, math.floor(n_estimators_minimum/5))) X = self.preprocess(X) n_estimator_increments = [n_estimators_final] # Very rough guess to size of a single tree before training if self.problem_type in [MULTICLASS, SOFTCLASS]: if self.num_classes is None: num_trees_per_estimator = 10 # Guess since it wasn't passed in, could also check y for a better value else: num_trees_per_estimator = self.num_classes else: num_trees_per_estimator = 1 bytes_per_estimator = num_trees_per_estimator * len(X) / 60000 * 1e6 # Underestimates by 3x on ExtraTrees available_mem = psutil.virtual_memory().available expected_memory_usage = bytes_per_estimator * n_estimators_final / available_mem expected_min_memory_usage = bytes_per_estimator * n_estimators_minimum / available_mem if expected_min_memory_usage > (0.5 * max_memory_usage_ratio): # if minimum estimated size is greater than 50% memory logger.warning(f'\tWarning: Model is expected to require {round(expected_min_memory_usage * 100, 2)}% of available memory (Estimated before training)...') raise NotEnoughMemoryError if n_estimators_final > n_estimators_test * 2: if self.problem_type == MULTICLASS: n_estimator_increments = [n_estimators_test, n_estimators_final] params['warm_start'] = True else: if expected_memory_usage > (0.05 * max_memory_usage_ratio): # Somewhat arbitrary, consider finding a better value, should it scale by cores? # Causes ~10% training slowdown, so try to avoid if memory is not an issue n_estimator_increments = [n_estimators_test, n_estimators_final] params['warm_start'] = True params['n_estimators'] = n_estimator_increments[0] if self._daal: if params.get('warm_start', False): params['warm_start'] = False model = model_cls(**params) time_train_start = time.time() for i, n_estimators in enumerate(n_estimator_increments): if i != 0: if params.get('warm_start', False): model.n_estimators = n_estimators else: params['n_estimators'] = n_estimators model = model_cls(**params) model =, y, sample_weight=sample_weight) if (i == 0) and (len(n_estimator_increments) > 1): time_elapsed = time.time() - time_train_start model_size_bytes = 0 for estimator in model.estimators_: # Uses far less memory than pickling the entire forest at once model_size_bytes += sys.getsizeof(pickle.dumps(estimator)) expected_final_model_size_bytes = model_size_bytes * (n_estimators_final / model.n_estimators) available_mem = psutil.virtual_memory().available model_memory_ratio = expected_final_model_size_bytes / available_mem ideal_memory_ratio = 0.15 * max_memory_usage_ratio n_estimators_ideal = min(n_estimators_final, math.floor(ideal_memory_ratio / model_memory_ratio * n_estimators_final)) if n_estimators_final > n_estimators_ideal: if n_estimators_ideal < n_estimators_minimum: logger.warning(f'\tWarning: Model is expected to require {round(model_memory_ratio*100, 2)}% of available memory...') raise NotEnoughMemoryError # don't train full model to avoid OOM error logger.warning(f'\tWarning: Reducing model \'n_estimators\' from {n_estimators_final} -> {n_estimators_ideal} due to low memory. Expected memory usage reduced from {round(model_memory_ratio*100, 2)}% -> {round(ideal_memory_ratio*100, 2)}% of available memory...') if time_limit is not None: time_expected = time_train_start - time_start + (time_elapsed * n_estimators_ideal / n_estimators) n_estimators_time = math.floor((time_limit - time_train_start + time_start) * n_estimators / time_elapsed) if n_estimators_time < n_estimators_ideal: if n_estimators_time < n_estimators_minimum: logger.warning(f'\tWarning: Model is expected to require {round(time_expected, 1)}s to train, which exceeds the maximum time limit of {round(time_limit, 1)}s, skipping model...') raise TimeLimitExceeded logger.warning(f'\tWarning: Reducing model \'n_estimators\' from {n_estimators_ideal} -> {n_estimators_time} due to low time. Expected time usage reduced from {round(time_expected, 1)}s -> {round(time_limit, 1)}s...') n_estimators_ideal = n_estimators_time for j in range(len(n_estimator_increments)): if n_estimator_increments[j] > n_estimators_ideal: n_estimator_increments[j] = n_estimators_ideal self.model = model self.params_trained['n_estimators'] = self.model.n_estimators # TODO: Remove this after simplifying _predict_proba to reduce code duplication. This is only present for SOFTCLASS support. def _predict_proba(self, X, **kwargs): X = self.preprocess(X, **kwargs) if self.problem_type == REGRESSION: return self.model.predict(X) elif self.problem_type == SOFTCLASS: return self.model.predict(X) elif self.problem_type == QUANTILE: return self.model.predict(X, quantile_levels=self.quantile_levels) y_pred_proba = self.model.predict_proba(X) return self._convert_proba_to_unified_form(y_pred_proba) def get_oof_pred_proba(self, X, normalize=None, **kwargs): """X should be the same X passed to `.fit`""" y_oof_pred_proba = self._get_oof_pred_proba(X=X, **kwargs) if normalize is None: normalize = self.normalize_pred_probas if normalize: y_oof_pred_proba = normalize_pred_probas(y_oof_pred_proba, self.problem_type) y_oof_pred_proba = y_oof_pred_proba.astype(np.float32) return y_oof_pred_proba # FIXME: Unknown if this works with quantile regression def _get_oof_pred_proba(self, X, y, **kwargs): if self._daal: raise AssertionError('DAAL forest backend does not support out-of-bag predictions.') if not self.model.bootstrap: raise ValueError('Forest models must set `bootstrap=True` to compute out-of-fold predictions via out-of-bag predictions.') # TODO: This can also be done via setting `oob_score=True` in model params, # but getting the correct `pred_time_val` that way is not easy, since we can't time the internal call. if (getattr(self.model, "oob_decision_function_", None) is None and getattr(self.model, "oob_prediction_", None) is None) \ and callable(getattr(self.model, "_set_oob_score", None)): X = self.preprocess(X) if getattr(self.model, "n_classes_", None) is not None: if self.model.n_outputs_ == 1: self.model.n_classes_ = [self.model.n_classes_] from sklearn.tree._tree import DTYPE, DOUBLE X, y = self.model._validate_data(X, y, multi_output=True, accept_sparse="csc", dtype=DTYPE) if y.ndim == 1: # reshape is necessary to preserve the data contiguity against vs # [:, np.newaxis] that does not. y = np.reshape(y, (-1, 1)) if getattr(y, "dtype", None) != DOUBLE or not y.flags.contiguous: y = np.ascontiguousarray(y, dtype=DOUBLE) self.model._set_oob_score(X, y) if getattr(self.model, "n_classes_", None) is not None: if self.model.n_outputs_ == 1: self.model.n_classes_ = self.model.n_classes_[0] if getattr(self.model, "oob_decision_function_", None) is not None: y_oof_pred_proba = self.model.oob_decision_function_ self.model.oob_decision_function_ = None # save memory elif getattr(self.model, "oob_prediction_", None) is not None: y_oof_pred_proba = self.model.oob_prediction_ self.model.oob_prediction_ = None # save memory else: raise AssertionError(f'Model class {type(self.model)} does not support out-of-fold prediction generation.') # TODO: Regression does not return NaN for missing rows, instead it sets them to 0. This makes life hard. # The below code corrects the missing rows to NaN instead of 0. # Don't bother if >60 trees, near impossible to have missing # If using 68% of data for training, chance of missing for each row is 1 in 11 billion. if self.problem_type == REGRESSION and self.model.n_estimators <= 60: from sklearn.ensemble._forest import _get_n_samples_bootstrap, _generate_unsampled_indices n_samples = len(y) n_predictions = np.zeros(n_samples) n_samples_bootstrap = _get_n_samples_bootstrap(n_samples, self.model.max_samples) for estimator in self.model.estimators_: unsampled_indices = _generate_unsampled_indices(estimator.random_state, n_samples, n_samples_bootstrap) n_predictions[unsampled_indices] += 1 missing_row_mask = n_predictions == 0 y_oof_pred_proba[missing_row_mask] = np.nan # fill missing prediction rows with average of non-missing rows if np.isnan(np.sum(y_oof_pred_proba)): if len(y_oof_pred_proba.shape) == 1: col_mean = np.nanmean(y_oof_pred_proba) y_oof_pred_proba[np.isnan(y_oof_pred_proba)] = col_mean else: col_mean = np.nanmean(y_oof_pred_proba, axis=0) inds = np.where(np.isnan(y_oof_pred_proba)) y_oof_pred_proba[inds] = np.take(col_mean, inds[1]) return self._convert_proba_to_unified_form(y_oof_pred_proba) # TODO: Add HPO def _hyperparameter_tune(self, **kwargs): return skip_hpo(self, **kwargs) def get_model_feature_importance(self): if self._features is None: # TODO: Consider making this raise an exception logger.warning('Warning: get_model_feature_importance called when self._features is None!') return dict() return dict(zip(self._features, self.model.feature_importances_)) def _get_default_auxiliary_params(self) -> dict: default_auxiliary_params = super()._get_default_auxiliary_params() extra_auxiliary_params = dict( ignored_type_group_raw=[R_OBJECT], ) default_auxiliary_params.update(extra_auxiliary_params) return default_auxiliary_params @classmethod def _get_default_ag_args_ensemble(cls, problem_type=None, **kwargs) -> dict: default_ag_args_ensemble = super()._get_default_ag_args_ensemble(problem_type=problem_type, **kwargs) if problem_type != QUANTILE: # use_child_oof not supported in quantile regression extra_ag_args_ensemble = {'use_child_oof': True} default_ag_args_ensemble.update(extra_ag_args_ensemble) return default_ag_args_ensemble def _more_tags(self): return {'valid_oof': True}