hyperparameter_hunter/environment.py
"""This module is central to the proper functioning of the entire library. It defines
:class:`Environment`, which (when activated) is used by the vast majority of the other
operation-critical modules in the library. :class:`Environment` can be viewed as a simple storage
container that defines settings that characterize the Experiments/OptimizationProtocols to be
conducted, and influence how those processes are carried out
Related
-------
:mod:`hyperparameter_hunter.settings`
This module is the doorway for other modules to access the settings defined by
:class:`environment.Environment`, which sets :attr:`hyperparameter_hunter.settings.G.Env` to
itself as its first action. This allows other modules to access any information they need from
the active :class:`environment.Environment` via :attr:`hyperparameter_hunter.settings.G.Env`.
:class:`hyperparameter_hunter.settings.G` also provides other modules with access to the
logging methods that are initialized by :class:`hyperparameter_hunter.environment.Environment`
Notes
-----
Despite the fact that :mod:`hyperparameter_hunter.settings` is the only module listed as being
"related", pretty much all the other modules in the library are related to
:class:`hyperparameter_hunter.environment.Environment` by way of this relation"""
##################################################
# Import Own Assets
##################################################
from hyperparameter_hunter.callbacks.bases import BaseCallback
from hyperparameter_hunter.i_o.reporting import ReportingHandler
from hyperparameter_hunter.keys.makers import CrossExperimentKeyMaker
from hyperparameter_hunter.metrics import format_metrics
from hyperparameter_hunter.sentinels import DatasetSentinel
from hyperparameter_hunter.settings import G, ASSETS_DIRNAME, RESULT_FILE_SUB_DIR_PATHS
from hyperparameter_hunter.utils.boltons_utils import remap
from hyperparameter_hunter.utils.file_utils import make_dirs, ParametersFromFile
from hyperparameter_hunter.utils.general_utils import Alias
from hyperparameter_hunter.utils.result_utils import format_predictions, default_do_full_save
##################################################
# Import Miscellaneous Assets
##################################################
from inspect import signature, isclass
import numpy as np
import os.path
import pandas as pd
from pandas.api.types import is_numeric_dtype
from typing import List, Optional, Tuple, Union
##################################################
# Import Learning Assets
##################################################
# noinspection PyProtectedMember
from sklearn.model_selection import _split as sk_cv
class Environment:
DEFAULT_PARAMS = dict(
environment_params_path=None,
results_path=None,
target_column="target",
id_column=None,
do_predict_proba=False,
prediction_formatter=format_predictions,
metrics=None,
metrics_params=dict(),
cv_type="KFold",
runs=1,
global_random_seed=32,
random_seeds=None,
random_seed_bounds=[0, 100_000],
cv_params=dict(),
verbose=3,
file_blacklist=None,
reporting_params=dict(
heartbeat_path=None, float_format="{:.5f}", console_params=None, heartbeat_params=None
),
to_csv_params=dict(),
do_full_save=default_do_full_save,
save_transformed_metrics=None,
)
@ParametersFromFile(key="environment_params_path", verbose=True)
@Alias("cv_type", ["cross_validation_type"])
@Alias("cv_params", ["cross_validation_params"])
@Alias("metrics", ["metrics_map"])
@Alias("reporting_params", ["reporting_handler_params"])
@Alias("results_path", ["root_results_path"])
def __init__(
self,
train_dataset, # TODO: Allow providing separate (train_input, train_target) dfs
environment_params_path=None,
*,
results_path=None,
metrics=None,
holdout_dataset=None, # TODO: Allow providing separate (holdout_input, holdout_target) dfs
test_dataset=None, # TODO: Allow providing separate (test_input, test_target) dfs
target_column=None,
id_column=None,
do_predict_proba=None,
prediction_formatter=None,
metrics_params=None,
cv_type=None,
runs=None,
global_random_seed=None,
random_seeds=None,
random_seed_bounds=None,
cv_params=None,
verbose=None,
file_blacklist=None,
reporting_params=None,
to_csv_params=None,
do_full_save=None,
experiment_callbacks=None,
experiment_recorders=None,
save_transformed_metrics=None,
):
"""Class to organize the parameters that allow Experiments/OptPros to be fairly compared
`Environment` is the collective starting point for all of HyperparameterHunter's biggest
and best toys: Experiments and OptimizationProtocols. Without an `Environment`, neither of
these will work.
The `Environment` is where we declare all the parameters that transcend traditional
"hyperparameters". It houses the stuff without which machine learning can't even really
start. Specifically, `Environment` cares about 1) The data used for fitting/predicting,
2) The cross-validation scheme used to split the data and fit models; and 3) How to evaluate
the predictions made on that data. There are plenty of other goodies documented below, but
the absolutely mission-critical parameters concerned with the above tasks are
`train_dataset`, `cv_type`, `cv_params`, and `metrics`. Additionally, it's important to
provide `results_path`, so Experiment/OptPro results can be saved, which is kind of what
HyperparameterHunter is all about
Parameters
----------
train_dataset: Pandas.DataFrame, or str path
The training data for the experiment. Will be split into train/holdout data, if
applicable, and train/validation data if cross-validation is to be performed. If str,
will attempt to read file at path via :func:`pandas.read_csv`. For more information on
which columns will be used during fitting/predicting, see the "Dataset columns" note
in the "Notes" section below
environment_params_path: String path, or None, default=None
If not None and is valid .json filepath containing an object (dict), the file's contents
are treated as the default values for all keys that match any of the below kwargs used
to initialize :class:`Environment`
results_path: String path, or None, default=None
If valid directory path and the results directory has not yet been created, it will be
created here. If this does not end with <ASSETS_DIRNAME>, it will be appended. If
<ASSETS_DIRNAME> already exists at this path, new results will also be stored here. If
None or invalid, results will not be stored
metrics: Dict, List, or None, default=None
Iterable describing the metrics to be recorded, along with a means to compute the value
of each metric. Should be of one of the two following forms:
List Form:
* ["<metric name>", "<metric name>", ...]:
Where each value is a string that names an attribute in :mod:`sklearn.metrics`
* [`Metric`, `Metric`, ...]:
Where each value of the list is an instance of :class:`metrics.Metric`
* [(<name>, <metric_function>, [<direction>]), (<\*args>), ...]:
Where each value of the list is a tuple of arguments that will be used to instantiate
a :class:`metrics.Metric`. Arguments given in tuples must be in order expected by
:class:`metrics.Metric`: (`name`, `metric_function`, `direction`)
Dict Form:
* {"<metric name>": <metric_function>, ...}:
Where each key is a name for the corresponding metric callable, which is used to
compute the value of the metric
* {"<metric name>": (<metric_function>, <direction>), ...}:
Where each key is a name for the corresponding metric callable and direction, all of
which are used to instantiate a :class:`metrics.Metric`
* {"<metric name>": "<sklearn metric name>", ...}:
Where each key is a name for the metric, and each value is the name of the attribute
in :mod:`sklearn.metrics` for which the corresponding key is an alias
* {"<metric name>": None, ...}:
Where each key is the name of the attribute in :mod:`sklearn.metrics`
* {"<metric name>": `Metric`, ...}:
Where each key names an instance of :class:`metrics.Metric`. This is the
internally-used format to which all other formats will be converted
Metric callable functions should expect inputs of form (target, prediction), and should
return floats. See the documentation of :class:`metrics.Metric` for information
regarding expected parameters and types
holdout_dataset: Pandas.DataFrame, callable, str path, or None, default=None
If pd.DataFrame, this is the holdout dataset. If callable, expects a function that takes
(self.train: DataFrame, self.target_column: str) as input and returns the new
(self.train: DataFrame, self.holdout: DataFrame). If str, will attempt to read file at
path via :func:`pandas.read_csv`. Else, there is no holdout set. For more information on
which columns will be used during fitting/predicting, see the "Dataset columns" note
in the "Notes" section below
test_dataset: Pandas.DataFrame, str path, or None, default=None
The testing data for the experiment. Structure should be identical to that of
`train_dataset`, except its `target_column` column can be empty or non-existent, because
`test_dataset` predictions will never be evaluated. If str, will attempt to read file at
path via :func:`pandas.read_csv`. For more information on which columns will be used
during fitting/predicting, see the "Dataset columns" note in the "Notes" section below
target_column: Str, or list, default='target'
If str, denotes the column name in all provided datasets (except test) that contains the
target output. If list, should be a list of strs designating multiple target columns.
For example, in a multi-class classification dataset like UCI's hand-written digits,
`target_column` would be a list containing ten strings. In this example, the
`target_column` data would be sparse, with a 1 to signify that a sample is a written
example of a digit (0-9). For a working example, see
'hyperparameter_hunter/examples/lib_keras_multi_classification_example.py'
id_column: Str, or None, default=None
If not None, str denoting the column name in all provided datasets containing sample IDs
do_predict_proba: Boolean, or int, default=False
* If False, :meth:`.models.Model.fit` will call :meth:`models.Model.model.predict`
* If True, it will call :meth:`models.Model.model.predict_proba`, and the values in all
columns will be used as the actual prediction values
* If `do_predict_proba` is an int, :meth:`.models.Model.fit` will call
:meth:`models.Model.model.predict_proba`, as is the case when `do_predict_proba` is
True, but the int supplied as `do_predict_proba` declares the column index to use as
the actual prediction values
* For example, for a model to call the `predict` method, `do_predict_proba=False`
(default). For a model to call the `predict_proba` method, and use all of the class
probabilities, `do_predict_proba=True`. To call the `predict_proba` method, and use
the class probabilities in the first column, `do_predict_proba=0`. To use the second
column (index 1) of the result, `do_predict_proba=1` - This often corresponds to the
positive class's probabilities in binary classification problems. To use the third
column `do_predict_proba=2`, and so on
prediction_formatter: Callable, or None, default=None
If callable, expected to have same signature as
:func:`.utils.result_utils.format_predictions`. That is, the callable will receive
(raw_predictions: np.array, dataset_df: pd.DataFrame, target_column: str,
id_column: str or None) as input and should return a properly formatted prediction
DataFrame. The callable uses raw_predictions as the content, dataset_df to provide any
id column, and target_column to identify the column in which to place raw_predictions
metrics_params: Dict, or None, default=dict()
Dictionary of extra parameters to provide to :meth:`.metrics.ScoringMixIn.__init__`.
`metrics` must be provided either 1) as an input kwarg to
:meth:`Environment.__init__` (see `metrics`), or 2) as a key in `metrics_params`,
but not both. An Exception will be raised if both are given, or if neither is given
cv_type: Class or str, default='KFold'
The class to define cross-validation splits. If str, it must be an attribute of
`sklearn.model_selection._split`, and it must be a cross-validation class that inherits
one of the following `sklearn` classes: `BaseCrossValidator`, or `_RepeatedSplits`.
Valid str values include 'KFold', and 'RepeatedKFold', although there are many more. It
must implement the following methods: [`__init__`, `split`]. If using a custom class,
see the following tested `sklearn` classes for proper implementations:
[`KFold`, `StratifiedKFold`, `RepeatedKFold`, `RepeatedStratifiedKFold`]. The arguments
provided to :meth:`cv_type.__init__` will be :attr:`Environment.cv_params`, which should
include the following: ['n_splits' <int>, 'n_repeats' <int> (if applicable)].
:meth:`cv_type.split` will receive the following arguments:
[:attr:`BaseExperiment.train_input_data`, :attr:`BaseExperiment.train_target_data`]
runs: Int, default=1
The number of times to fit a model within each fold to perform multiple-run-averaging
with different random seeds
global_random_seed: Int, default=32
The initial random seed used just before generating an Experiment's random_seeds. This
ensures consistency for `random_seeds` between Experiments, without having to explicitly
provide it here
random_seeds: None, or List, default=None
If None, `random_seeds` of the appropriate shape will be created automatically. Else,
must be a list of ints of shape (`cv_params['n_repeats']`, `cv_params['n_splits']`,
`runs`). If `cv_params` does not have the key `n_repeats` (because standard
cross-validation is being used), the value will default to 1. See
:meth:`.experiments.BaseExperiment._random_seed_initializer` for info on expected shape
random_seed_bounds: List, default=[0, 100000]
A list containing two integers: the lower and upper bounds, respectively, for generating
an Experiment's random seeds in
:meth:`.experiments.BaseExperiment._random_seed_initializer`. Generally, leave this
kwarg alone
cv_params: dict, or None, default=dict()
Parameters provided upon initialization of cv_type. Keys may be any args accepted by
:meth:`cv_type.__init__`. Number of fold splits must be provided via "n_splits", and
number of repeats (if applicable for `cv_type`) must be provided via "n_repeats"
verbose: Int, boolean, default=3
Verbosity of printing for any experiments performed while this Environment is active
Higher values indicate more frequent logging. Logs are still recorded in the heartbeat
file regardless of verbosity level. `verbose` only dictates which logs are visible in
the console. The following table illustrates which types of logging messages will be
visible with each verbosity level::
| Verbosity | Keys/IDs | Final Score | Repetitions* | Folds | Runs* | Run Starts* | Result Files | Other |
|:---------:|:--------:|:-----------:|:------------:|:-----:|:-----:|:-----------:|:------------:|:-----:|
| 0 | | | | | | | | |
| 1 | Yes | Yes | | | | | | |
| 2 | Yes | Yes | Yes | Yes | | | | |
| 3 | Yes | Yes | Yes | Yes | Yes | | | |
| 4 | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes |
*\*: If such logging is deemed appropriate with the given cross-validation parameters. In
other words, repetition/run logging will only be verbose if Environment was given more
than one repetition/run, respectively*
file_blacklist: List of str, or None, or 'ALL', default=None
If list of str, the result files named within are not saved to their respective
directory in "<ASSETS_DIRNAME>/Experiments". If None, all result files are saved.
If 'ALL', nothing at all will be saved for the Experiments. If the path of the file that
initializes an Experiment does not end with a ".py" extension, the Experiment proceeds
as if "script_backup" had been added to `file_blacklist`. This means that backup files
will not be created for Jupyter notebooks (or any other non-".py" files). For info on
acceptable values, see :func:`validate_file_blacklist`
reporting_params: Dict, default=dict()
Parameters passed to initialize :class:`.reporting.ReportingHandler`
to_csv_params: Dict, default=dict()
Parameters passed to the calls to :meth:`pandas.frame.DataFrame.to_csv` in
:mod:`recorders`. In particular, this is where an Experiment's final prediction files
are saved, so the values here will affect the format of the .csv prediction files.
Warning: If `to_csv_params` contains the key "path_or_buf", it will be removed.
Otherwise, all items are supplied directly to :meth:`to_csv`, including kwargs it might
not be expecting if they are given
do_full_save: None, or callable, default=:func:`utils.result_utils.default_do_full_save`
If callable, expected to take an Experiment's result description dict as input and
return a boolean. If None, treated as a callable that returns True. This parameter is
used by :class:`recorders.DescriptionRecorder` to determine whether the Experiment
result files following the description should also be created. If `do_full_save` returns
False, result file-saving is stopped early, and only the description is saved. If
`do_full_save` returns True, all files not in `file_blacklist` are saved normally. This
allows you to skip creation of an Experiment's predictions, logs, and heartbeats if its
score does not meet some threshold you set, for example. `do_full_save` receives the
Experiment description dict as input, so for help setting `do_full_save`, just look into
one of your Experiment descriptions
experiment_callbacks: `LambdaCallback`, or list of `LambdaCallback` (optional)
Callbacks injected directly into Experiments, adding new functionality, or customizing
existing processes. Should be a :class:`LambdaCallback` or a list of such classes.
`LambdaCallback` can be created using :func:`.callbacks.bases.lambda_callback`, which
documents the options for creating callbacks. `experiment_callbacks` will be added to
the MRO of the executed Experiment class by :class:`.experiment_core.ExperimentMeta` at
`__call__` time, making `experiment_callbacks` new base classes of the Experiment. See
:func:`.callbacks.bases.lambda_callback` for more information. Note that the Experiments
conducted by OptPros will still benefit from `experiment_callbacks`. The presence of
LambdaCallbacks will affect neither Environment keys, nor Experiment keys. In other
words, for the purposes of Experiment matching/recording, all other factors being equal,
an Experiment with `experiment_callbacks` is considered identical to an Experiment
without, despite whatever custom functionality was added by the LambdaCallbacks
experiment_recorders: List, None, default=None
If not None, may be a list whose values are tuples of
(<:class:`recorders.BaseRecorder` descendant>, <str result_path>). The result_path str
should be a path relative to `results_path` that specifies the directory/file in
which the product of the custom recorder should be saved. The contents of
`experiment_recorders` will be provided to `recorders.RecorderList` upon completion of
an Experiment, and, if the subclassing documentation in `recorders` is followed
properly, will create or update a result file for the just-executed Experiment
save_transformed_metrics: Boolean (optional)
Declares manner in which a model's predictions should be evaluated through the provided
`metrics`, with regard to target data transformations. This setting can be ignored if
no transformation of the target variable takes place (either through
:class:`~hyperparameter_hunter.feature_engineering.FeatureEngineer`,
:class:`~hyperparameter_hunter.feature_engineering.EngineerStep`, or otherwise).
The default value of `save_transformed_metrics` depends on the dtype of the target data
in `train_dataset`. If all target columns are numeric, `save_transformed_metrics`=False,
meaning metric evaluation should use the original/inverted targets and predictions. Else
if any target column is non-numeric, `save_transformed_metrics`=True, meaning evaluation
should use the transformed targets and predictions because most metrics require numeric
inputs. This is described further in :attr:`save_transformed_metrics`. A more
descriptive name for this may be "calculate_metrics_using_transformed_predictions",
but that's a bit verbose--even by my standards
Other Parameters
----------------
cross_validation_type: ...
* Alias for `cv_type` *
cross_validation_params: ...
* Alias for `cv_params` *
metrics_map: ...
* Alias for `metrics` *
reporting_handler_params: ...
* Alias for `reporting_params` *
root_results_path: ...
* Alias for `results_path` *
Attributes
----------
train_input: DatasetSentinel
Sentinel replaced with current train input data during `Model` fitting/predicting.
Commonly given in the `model_extra_params` kwargs of
:class:`hyperparameter_hunter.experiments.BaseExperiment` or
:meth:`hyperparameter_hunter.optimization.protocol_core.BaseOptPro.forge_experiment` for
`eval_set`-like hyperparameters. Importantly, the actual value of this Sentinel is
determined after performing cross-validation data splitting, and after executing
:class:`~hyperparameter_hunter.feature_engineering.FeatureEngineer`
train_target: DatasetSentinel
Like :attr:`.train_input`, except for current train target data
validation_input: DatasetSentinel
Like :attr:`.train_input`, except for current validation input data
validation_target: DatasetSentinel
Like :attr:`.train_input`, except for current validation target data
holdout_input: DatasetSentinel
Like :attr:`.train_input`, except for current holdout input data
holdout_target: DatasetSentinel
Like :attr:`.train_input`, except for current holdout target data
Notes
-----
Dataset columns: In order to specify the columns to be used by the three dataset kwargs
(`train_dataset`, `holdout_dataset`, `test_dataset`) during fitting and predicting, a few
attributes can be used. On `Environment` initialization, the columns specified by the
following kwargs will be separated from the rest of the dataset during training/predicting:
1) `target_column`, which names the column containing the target output labels for the input
data; and 2) `id_column`, which (if given) represents the name of the column that contains
identifying information for each data sample, and should otherwise have no relation to the
actual data. Additionally, the `feature_selector` kwarg of the descendants of
:class:`hyperparameter_hunter.experiments.BaseExperiment` (like
:class:`hyperparameter_hunter.experiments.CVExperiment`) is used to filter out
columns of the given datasets prior to fitting. See its documentation for more information,
but it can effectively be used to remove any columns from the datasets
Overriding default kwargs at `environment_params_path`: If you have any of the above kwargs
specified in the .json file at environment_params_path (except environment_params_path,
which will be ignored), you can override its value by passing it as a kwarg when
initializing :class:`Environment`. The contents at environment_params_path are only used
when the matching kwarg supplied at initialization is None. See
"/examples/environment_params_path_example.py" for details
The order of precedence for determining the value of each parameter is as follows, with
items at the top having the highest priority, and deferring only to the items below if
their own value is None:
* 1)kwargs passed directly to :meth:`.Environment.__init__` on initialization,
* 2)keys of the file at environment_params_path (if valid .json object),
* 3)keys of :attr:`hyperparameter_hunter.environment.Environment.DEFAULT_PARAMS`
do_predict_proba: Because this parameter can be either a boolean or an integer, it is
important to explicitly pass booleans rather than truthy or falsey values. Similarly, only
pass integers if you intend for the value to be used as a column index. Do not pass `0` to
mean `False`, or `1` to mean `True`
"""
G.Env = self
self.environment_params_path = environment_params_path
self.results_path = results_path
#################### Attributes Used by Experiments ####################
self.target_column = target_column
self.id_column = id_column
self.train_dataset = train_dataset
self.holdout_dataset = holdout_dataset
self.test_dataset = test_dataset
self.do_predict_proba = do_predict_proba
self.prediction_formatter = prediction_formatter
self.metrics = metrics
self.metrics_params = metrics_params
self.cv_type = cv_type
self.runs = runs
self.global_random_seed = global_random_seed
self.random_seeds = random_seeds
self.random_seed_bounds = random_seed_bounds
self.cv_params = cv_params
# TODO: Make `cv_params` optional. If not given, use default `cv_type` parameters. Test
# Env keys are identical without `cv_params` and when explicitly given default params
#################### Ancillary Environment Settings ####################
self.verbose = verbose
self.file_blacklist = file_blacklist
self.reporting_params = reporting_params or {}
self.to_csv_params = to_csv_params or {}
self.do_full_save = do_full_save
self.experiment_callbacks = experiment_callbacks or []
self.experiment_recorders = experiment_recorders or []
self.save_transformed_metrics = save_transformed_metrics
self.result_paths = {
"root": self.results_path,
"checkpoint": None,
"description": None,
"heartbeat": None,
"predictions_holdout": None,
"predictions_in_fold": None,
"predictions_oof": None,
"predictions_test": None,
"script_backup": None,
"tested_keys": None,
"key_attribute_lookup": None,
"leaderboards": None,
"global_leaderboard": None,
"current_heartbeat": None,
}
self.current_task = None
self.cross_experiment_key = None
self.environment_workflow()
def __repr__(self):
return f"{self.__class__.__name__}(cross_experiment_key={self.cross_experiment_key!s})"
def __eq__(self, other):
return self.cross_experiment_key == other
# def __enter__(self):
# pass
# def __exit__(self):
# G.reset_attributes()
##################################################
# Properties
##################################################
#################### `results_path` ####################
@property
def results_path(self) -> Optional[str]:
return self._results_path
@results_path.setter
def results_path(self, value):
self._results_path = value
if self._results_path is None:
G.warn("Received results_path=None. Results will not be stored at all.")
elif isinstance(self._results_path, str):
if not self._results_path.endswith(ASSETS_DIRNAME):
self._results_path = os.path.join(self._results_path, ASSETS_DIRNAME)
# self.result_paths["root"] = self.results_path
if not os.path.exists(self._results_path):
make_dirs(self._results_path, exist_ok=True)
else:
raise TypeError(f"results_path must be None or str, not {value}")
#################### `target_column` ####################
@property
def target_column(self) -> list:
return self._target_column
@target_column.setter
def target_column(self, value):
if value is None:
# TODO: Clean up this logic duplicated by `update_custom_environment_params`' setting of
# `target_column` to its default -- `save_transformed_metrics` needs it on __init__
value = [self.DEFAULT_PARAMS["target_column"]]
self._target_column = [value] if isinstance(value, str) else value
#################### `train_dataset` ####################
@property
def train_dataset(self) -> pd.DataFrame:
return self._train_dataset
@train_dataset.setter
def train_dataset(self, value):
self._train_dataset = pd.read_csv(value) if isinstance(value, str) else value
#################### `test_dataset` ####################
@property
def test_dataset(self) -> Optional[pd.DataFrame]:
return self._test_dataset
@test_dataset.setter
def test_dataset(self, value):
self._test_dataset = pd.read_csv(value) if isinstance(value, str) else value
#################### `holdout_dataset` ####################
@property
def holdout_dataset(self) -> Optional[pd.DataFrame]:
return self._holdout_dataset
@holdout_dataset.setter
def holdout_dataset(self, value):
self._train_dataset, self._holdout_dataset = define_holdout_set(
self.train_dataset, value, self.target_column
)
#################### `file_blacklist` ####################
@property
def file_blacklist(self) -> Union[list, str]:
return self._file_blacklist
@file_blacklist.setter
def file_blacklist(self, value):
self._file_blacklist = validate_file_blacklist(value)
if self.results_path is None:
self._file_blacklist = "ALL"
#################### `metrics_params`/`metrics` ####################
# TODO: Move their validations to properties here
#################### `cv_type` ####################
@property
def cv_type(self) -> type:
return self._cv_type
@cv_type.setter
def cv_type(self, value):
if isinstance(value, str):
try:
self._cv_type = sk_cv.__getattribute__(value)
except AttributeError:
raise AttributeError(f"'{value}' not in `sklearn.model_selection._split`")
else: # Assumed to be a valid CV class
self._cv_type = value
#################### `to_csv_params` ####################
@property
def to_csv_params(self) -> dict:
return self._to_csv_params
@to_csv_params.setter
def to_csv_params(self, value):
self._to_csv_params = {k: v for k, v in value.items() if k != "path_or_buf"}
#################### `cross_experiment_params` ####################
@property
def cross_experiment_params(self) -> dict:
return dict(
cv_type=self.cv_type,
runs=self.runs,
global_random_seed=self.global_random_seed,
random_seeds=self.random_seeds,
random_seed_bounds=self.random_seed_bounds,
)
#################### `experiment_callbacks` ####################
@property
def experiment_callbacks(self) -> list:
return self._experiment_callbacks
@experiment_callbacks.setter
def experiment_callbacks(self, value):
if not isinstance(value, list):
self._experiment_callbacks = [value]
else:
self._experiment_callbacks = value
for cb in self._experiment_callbacks:
if not isclass(cb):
raise TypeError(f"experiment_callbacks must be classes, not {type(cb)}: {cb}")
if issubclass(cb, BaseCallback):
continue
if cb.__name__ != "LambdaCallback":
raise ValueError(f"experiment_callbacks must be LambdaCallback instances, not {cb}")
#################### `save_transformed_metrics` ####################
@property
def save_transformed_metrics(self) -> bool:
"""If `save_transformed_metrics` is True, and target transformation does occur, then
experiment metrics are calculated using the transformed targets and predictions, which is
the form returned directly by a fitted model's `predict` method. For example, if target data
is label-encoded, and an :class:`feature_engineering.EngineerStep` is used to one-hot encode
the target, then metrics functions will receive the following as input:
(one-hot-encoded targets, one-hot-encoded predictions).
Conversely, if `save_transformed_metrics` is False, and target transformation does occur,
then experiment metrics are calculated using the inverse of the transformed targets and
predictions, which is same form as the original target data. Continuing the example of
label-encoded target data, and an :class:`feature_engineering.EngineerStep` to one-hot
encode the target, in this case, metrics functions will receive the following as input:
(label-encoded targets, label-encoded predictions)"""
return self._save_transformed_metrics
@save_transformed_metrics.setter
def save_transformed_metrics(self, value):
if value is None:
if all(is_numeric_dtype(self.train_dataset[col]) for col in self.target_column):
# If all target columns are numeric, assume metric evaluation should use the
# original/inverted targets and predictions
value = False
else:
# If any target column is non-numeric, assume evaluation should use the transformed
# targets and predictions because most metrics require numeric inputs
value = True
self._save_transformed_metrics = value
##################################################
# Core Methods
##################################################
def environment_workflow(self):
"""Execute all methods required to validate the environment and run Experiments"""
self.update_custom_environment_params()
self.validate_parameters()
self.format_result_paths()
self.generate_cross_experiment_key()
G.log("Cross-Experiment Key: '{!s}'".format(self.cross_experiment_key))
def validate_parameters(self):
"""Ensure the provided parameters are valid and properly formatted"""
#################### metrics_params/metrics ####################
if (self.metrics is not None) and ("metrics" in self.metrics_params.keys()):
raise ValueError(
"`metrics` may be provided as a kwarg, or as a `metrics_params` key, but NOT BOTH. Received: "
+ f"\n `metrics`={self.metrics}\n `metrics_params`={self.metrics_params}"
)
else:
_metrics_alias = "metrics"
if self.metrics is None:
try:
self.metrics = self.metrics_params["metrics"]
except KeyError:
self.metrics = self.metrics_params["metrics_map"]
_metrics_alias = "metrics_map"
self.metrics = format_metrics(self.metrics)
self.metrics_params = {**{_metrics_alias: self.metrics}, **self.metrics_params}
def format_result_paths(self):
"""Remove paths contained in file_blacklist, and format others to prepare for saving results"""
if self.file_blacklist == "ALL" or self.results_path is None:
return
# Blacklist the prediction files for any datasets that were not given
if self.holdout_dataset is None:
self.file_blacklist.append("predictions_holdout")
if self.test_dataset is None:
self.file_blacklist.append("predictions_test")
# Add given `experiment_recorders` to `result_paths`
for recorder in self.experiment_recorders:
try:
recorder, result_path = recorder
except IndexError:
raise IndexError(f"Expected `recorder` to be tuple of (class, str), not {recorder}")
self.result_paths[recorder.result_path_key] = result_path
# Set full filepath for result files relative to `results_path`, or to None (blacklist)
for k in self.result_paths.keys():
if k == "root":
continue
elif k not in self.file_blacklist:
# If `k` not in `RESULT_FILE_SUB_DIR_PATHS`, then added via `experiment_recorders`
self.result_paths[k] = os.path.join(
self.results_path, RESULT_FILE_SUB_DIR_PATHS.get(k, self.result_paths[k])
)
else:
self.result_paths[k] = None
# G.debug('Result file "{}" has been blacklisted'.format(k))
def update_custom_environment_params(self):
"""Try to update null parameters from environment_params_path, or DEFAULT_PARAMS"""
allowed_parameter_keys = [
k for k, v in signature(Environment).parameters.items() if v.kind == v.KEYWORD_ONLY
]
for k in allowed_parameter_keys:
if getattr(self, k) is None:
setattr(self, k, self.DEFAULT_PARAMS.get(k, None))
def generate_cross_experiment_key(self):
"""Generate a key to describe the current Environment's cross-experiment parameters"""
parameters = dict(
metrics_params=self.metrics_params,
cv_params=self.cv_params,
target_column=self.target_column,
id_column=self.id_column,
do_predict_proba=self.do_predict_proba,
prediction_formatter=self.prediction_formatter,
train_dataset=self.train_dataset,
test_dataset=self.test_dataset,
holdout_dataset=self.holdout_dataset,
cross_experiment_params=self.cross_experiment_params.copy(),
to_csv_params=self.to_csv_params,
)
#################### Revert Aliases for Compatibility ####################
# If any aliases were used during call to `Environment.__init__`, replace the default names
# in `parameters` with the alias used. This ensures compatibility with Environment keys
# made in earlier versions
aliases_used = getattr(self, "__hh_aliases_used", {})
# noinspection PyUnusedLocal
def _visit(path, key, value):
if key in aliases_used:
key = aliases_used.pop(key)
return (key, value)
if aliases_used:
parameters = remap(parameters, visit=_visit)
#################### Make `cross_experiment_key` ####################
self.cross_experiment_key = CrossExperimentKeyMaker(parameters)
def initialize_reporting(self):
"""Initialize reporting for the Environment and Experiments conducted during its lifetime"""
reporting_params = self.reporting_params
reporting_params["heartbeat_path"] = self.result_paths["current_heartbeat"]
reporting_handler = ReportingHandler(**reporting_params)
#################### Make Unified Logging Globally Available ####################
G.log = reporting_handler.log
G.debug = reporting_handler.debug
G.warn = reporting_handler.warn
##################################################
# Dataset Sentinels for Use as Extra Parameters
##################################################
@property
def train_input(self):
"""Get a `DatasetSentinel` representing an Experiment's `fold_train_input`
Returns
-------
DatasetSentinel:
A `Sentinel` that will be converted to :attr:`hyperparameter_hunter.experiments.BaseExperiment.fold_train_input` upon
`Model` initialization"""
return DatasetSentinel("train_input", **self._dataset_sentinel_helper())
@property
def train_target(self):
"""Get a `DatasetSentinel` representing an Experiment's `fold_train_target`
Returns
-------
DatasetSentinel:
A `Sentinel` that will be converted to :attr:`hyperparameter_hunter.experiments.BaseExperiment.fold_train_target` upon
`Model` initialization"""
return DatasetSentinel("train_target", **self._dataset_sentinel_helper())
@property
def validation_input(self):
"""Get a `DatasetSentinel` representing an Experiment's `fold_validation_input`
Returns
-------
DatasetSentinel:
A `Sentinel` that will be converted to :attr:`hyperparameter_hunter.experiments.BaseExperiment.fold_validation_input`
upon `Model` initialization"""
return DatasetSentinel("oof_input", **self._dataset_sentinel_helper())
@property
def validation_target(self):
"""Get a `DatasetSentinel` representing an Experiment's `fold_validation_target`
Returns
-------
DatasetSentinel:
A `Sentinel` that will be converted to :attr:`hyperparameter_hunter.experiments.BaseExperiment.fold_validation_target`
upon `Model` initialization"""
return DatasetSentinel("oof_target", **self._dataset_sentinel_helper())
@property
def holdout_input(self):
"""Get a `DatasetSentinel` representing an Experiment's `holdout_input_data`
Returns
-------
DatasetSentinel:
A `Sentinel` that will be converted to :attr:`hyperparameter_hunter.experiments.BaseExperiment.holdout_input_data`
upon `Model` initialization"""
return DatasetSentinel(
"holdout_input", self.cross_experiment_key.parameters["holdout_dataset"]
)
@property
def holdout_target(self):
"""Get a `DatasetSentinel` representing an Experiment's `holdout_target_data`
Returns
-------
DatasetSentinel:
A `Sentinel` that will be converted to :attr:`hyperparameter_hunter.experiments.BaseExperiment.holdout_target_data`
upon `Model` initialization"""
return DatasetSentinel(
"holdout_target", self.cross_experiment_key.parameters["holdout_dataset"]
)
def _dataset_sentinel_helper(self):
"""Helper method for retrieving train/validation sentinel parameters"""
params = self.cross_experiment_key.parameters
return dict(
dataset_hash=params["train_dataset"],
cv_type=params["cross_experiment_params"].get(
"cv_type", params["cross_experiment_params"].get("cross_validation_type", None)
),
global_random_seed=params["cross_experiment_params"]["global_random_seed"],
random_seeds=params["cross_experiment_params"]["random_seeds"],
)
def define_holdout_set(
train_set: pd.DataFrame,
holdout_set: Union[pd.DataFrame, callable, str, None],
target_column: Union[str, List[str]],
) -> Tuple[pd.DataFrame, Optional[pd.DataFrame]]:
"""Create `holdout_set` (if necessary) by loading a DataFrame from a .csv file, or by separating
`train_set`, and return the updated (`train_set`, `holdout_set`) pair
Parameters
----------
train_set: Pandas.DataFrame
Training DataFrame. Will be split into train/holdout data, if `holdout_set` is callable
holdout_set: Pandas.DataFrame, callable, str, or None
If pd.DataFrame, this is the holdout dataset. If callable, expects a function that takes
(`train_set`, `target_column`) as input and returns the new (`train_set`, `holdout_set`). If
str, will attempt to read file at path via :func:`pandas.read_csv`. Else, no holdout set
target_column: Str, or list
If str, denotes the column name in provided datasets that contains the target output. If
list, should be a list of strs designating multiple target columns
Returns
-------
train_set: Pandas.DataFrame
`train_set` if `holdout_set` is not callable. Else `train_set` modified by `holdout_set`
holdout_set: Pandas.DataFrame, or None
Original DataFrame, or DataFrame read from str filepath, or a portion of `train_set` if
`holdout_set` is callable, or None"""
#################### Update `holdout_set` ####################
if callable(holdout_set):
train_set, holdout_set = holdout_set(train_set, target_column)
elif isinstance(holdout_set, str):
holdout_set = pd.read_csv(holdout_set)
#################### Validate `holdout_set` ####################
try:
if holdout_set is None or np.array_equal(train_set.columns, holdout_set.columns):
return train_set, holdout_set
except AttributeError:
raise TypeError(f"holdout_set must be None, DataFrame, callable, or str, not {holdout_set}")
raise ValueError(f"Mismatched columns\n{train_set.columns}\n!=\n{holdout_set.columns}")
##################################################
# File Blacklist Utilities
##################################################
def validate_file_blacklist(blacklist):
"""Validate contents of blacklist. For most values, the corresponding file is saved upon
completion of the experiment. See the "Notes" section below for details on some special cases
Parameters
----------
blacklist: List of strings, or None
The result files that should not be saved
Returns
-------
blacklist: List
If not empty, acceptable list of result file types to blacklist
Notes
-----
'heartbeat': If the heartbeat file is saved, a new file is not generated and saved to the
"Experiments/Heartbeats" directory as is the case with most other files. Instead, the general
"Heartbeat.log" file is copied and renamed to the current experiment id, then saved to the
appropriate dir. This is because the general "Heartbeat.log" file represents the heartbeat
for whatever experiment is currently in progress.
'script_backup': This file is saved as quickly as possible after starting a new experiment,
rather than waiting for the experiment to end. There are two reasons for this behavior: 1) to
avoid saving any changes that may have been made to a file after it has been executed, and 2)
to have the offending file in the event of a catastrophic failure that results in no other
files being saved. As stated in the documentation of the `file_blacklist` parameter of
`Environment`, if the path of the file that initializes an Experiment does not end with a ".py"
extension, the Experiment proceeds as if "script_backup" had been added to `blacklist`. This
means that backup files will not be created for Jupyter notebooks (or any other non-".py" files)
'description' and 'tested_keys': These two results types constitute a bare minimum of sorts for
experiment recording. If either of these two are blacklisted, then as far as the library is
concerned, the experiment never took place.
'tested_keys' (continued): If this string is included in the blacklist, then the contents of the
"KeyAttributeLookup" directory will also be excluded from the list of files to update
'current_heartbeat': The general heartbeat file that should be stored at
'HyperparameterHunterAssets/Heartbeat.log'. If this value is blacklisted, then 'heartbeat' is
also added to `blacklist` automatically out of necessity. This is done because the heartbeat
file for the current experiment cannot be created as a copy of the general heartbeat file if the
general heartbeat file is never created in the first place"""
valid_values = [
# 'checkpoint',
"description",
"heartbeat",
"predictions_holdout",
"predictions_in_fold",
"predictions_oof",
"predictions_test",
"script_backup",
"tested_keys",
"current_heartbeat",
]
if blacklist == "ALL":
G.warn('WARNING: Received `blacklist`="ALL". Nothing will be saved')
return blacklist
if not blacklist:
return []
elif not isinstance(blacklist, list):
raise TypeError("Expected blacklist to be a list, not: {}".format(blacklist))
elif not all([isinstance(_, str) for _ in blacklist]):
invalid_files = [(type(_).__name__, _) for _ in blacklist if not isinstance(_, str)]
raise TypeError("Expected blacklist contents to be strings, not: {}".format(invalid_files))
for a_file in blacklist:
if a_file not in valid_values:
raise ValueError(f"Invalid blacklist value: {a_file}.\nExpected one of: {valid_values}")
if a_file in ["description", "tested_keys"]:
G.warn(f"Including {a_file!r} in blacklist will severely impede library functionality")
# Blacklist experiment-specific heartbeat if general (current) heartbeat is blacklisted
if ("current_heartbeat" in blacklist) and ("heartbeat" not in blacklist):
blacklist.append("heartbeat")
return blacklist