hyperparameter_hunter/i_o/reporting.py
##################################################
# Import Own Assets
##################################################
from hyperparameter_hunter.i_o import exceptions
from hyperparameter_hunter.space.dimensions import Categorical
from hyperparameter_hunter.settings import G
from hyperparameter_hunter.utils.general_utils import now_time, expand_mins_secs
##################################################
# Import Miscellaneous Assets
##################################################
from contextlib import suppress
from datetime import datetime
import inspect
import logging
import os.path
import sys
from typing import List
class ReportingHandler(object):
def __init__(
self,
heartbeat_path=None,
float_format="{:.5f}",
console_params=None,
heartbeat_params=None,
add_frame=False,
):
"""Class in control of logging methods, log formatting, and initializing Experiment logging
Parameters
----------
heartbeat_path: Str path, or None, default=None
If string and valid heartbeat path, logging messages will also be saved in this file
float_format: String, default='{:.5f}'
If not default, must be a valid formatting string for floating point values. If invalid,
default will be used
console_params: Dict, or None, default=None
Parameters passed to :meth:`_configure_console_handler`
heartbeat_params: Dict, or None, default=None
Parameters passed to :meth:`_configure_heartbeat_handler`
add_frame: Boolean, default=False
If True, whenever :meth:`log` is called, the source of the call will be prepended to
the content being logged"""
self.reporting_type = "logging" # TODO: Add `reporting_type` kwarg (logging, advanced)
self.heartbeat_path = heartbeat_path
self.float_format = float_format
self.console_params = console_params or {}
self.heartbeat_params = heartbeat_params or {}
self.add_frame = add_frame
self._validate_parameters()
self._configure_reporting_type()
def _validate_parameters(self):
"""Ensure all logging parameters are properly formatted"""
#################### reporting_type ####################
valid_types = ["logging", "standard", "advanced"]
if not isinstance(self.reporting_type, str):
raise TypeError(f"reporting_type must be a str. Received {self.reporting_type}")
if self.reporting_type not in valid_types:
raise ValueError(f"reporting_type must be in {valid_types}, not {self.reporting_type}")
#################### heartbeat_path ####################
if self.heartbeat_path is not None:
if not isinstance(self.heartbeat_path, str):
raise TypeError(f"heartbeat_path must be a str. Received {self.heartbeat_path}")
head, tail = os.path.split(self.heartbeat_path)
if not tail.endswith(".log"):
raise ValueError(f"heartbeat_path must end in '.log'. Given {self.heartbeat_path}")
if not os.path.exists(head):
raise FileNotFoundError(
f"heartbeat_path must start with an existing dir. Given {self.heartbeat_path}"
)
#################### float_format ####################
if not isinstance(self.float_format, str):
raise TypeError(f"float_format must be a format str. Received {self.float_format}")
if (not self.float_format.startswith("{")) or (not self.float_format.endswith("}")):
raise ValueError(f"float_format must be inside '{{' and '}}'. Got {self.float_format}")
#################### console_params ####################
if not isinstance(self.console_params, dict):
raise TypeError(f"console_params must be dict or None. Given {self.console_params}")
#################### heartbeat_params ####################
if not isinstance(self.heartbeat_params, dict):
raise TypeError(f"heartbeat_params must be dict or None. Given {self.heartbeat_params}")
def _configure_reporting_type(self):
"""Set placeholder logging methods to :attr:`reporting_type` specs and initialize logging"""
if self.reporting_type == "standard":
raise ValueError("Standard logging is not yet implemented. Please choose 'logging'")
# setattr(self, 'log', self._standard_log)
# setattr(self, 'debug', self._standard_debug)
# setattr(self, 'warn', self._standard_warn)
elif self.reporting_type == "logging":
setattr(self, "log", self._logging_log)
setattr(self, "debug", self._logging_debug)
setattr(self, "warn", self._logging_warn)
self._initialize_logging_logging()
elif self.reporting_type == "advanced":
raise ValueError("Advanced logging unimplemented. Please use 'logging'")
def _initialize_logging_logging(self):
"""Initialize and configure logging to be handled by the `logging` library"""
#################### Clear Logging Configuration ####################
root = logging.getLogger()
list(map(root.removeHandler, root.handlers[:]))
list(map(root.removeFilter, root.filters[:]))
#################### Configure Logging ####################
exceptions.hook_exception_handler()
_logger = logging.getLogger(__name__)
_logger.setLevel(logging.DEBUG)
handlers = [self._configure_console_handler(**self.console_params)]
# Suppress FileExistsError - Raised when self.heartbeat_path is None, meaning heartbeat blacklisted
with suppress(FileExistsError):
handlers.append(self._configure_heartbeat_handler(**self.heartbeat_params))
logging.basicConfig(handlers=handlers, level=logging.DEBUG)
self.debug("Logging Logging has been initialized!")
# noinspection PyUnusedLocal
@staticmethod
def _configure_console_handler(level="INFO", fmt=None, datefmt="%H:%M:%S", style="%", **kwargs):
"""Configure the console handler in charge of printing log messages
Parameters
----------
level: String, or Int, default='DEBUG'
Minimum message level for the console. Passed to :meth:`logging.StreamHandler.setlevel`
fmt: String, or None, default=None
Message formatting string for the console. Passed to :meth:`logging.Formatter.__init__`
datefmt: String, or None, default="%H:%M:%S"
Date formatting string for the console. Passed to :meth:`logging.Formatter.__init__`.
For the `logging` library default, use `datefmt=None` ("%Y-%m-%d %H:%M:%S" + <ms>)
style: String, default='%'
Type of string formatting used. Passed to :meth:`logging.Formatter.__init__`
**kwargs: Dict
Extra keyword arguments
Returns
-------
console_handler: `logging.StreamHandler` instance
The instantiated handler for the console"""
console_handler = logging.StreamHandler(stream=sys.stdout)
console_handler.setLevel(level)
fmt = fmt or "<%(asctime)s> %(message)s"
formatter = logging.Formatter(fmt=fmt, datefmt=datefmt, style=style)
console_handler.setFormatter(formatter)
return console_handler
# noinspection PyUnusedLocal
def _configure_heartbeat_handler(
self, level="DEBUG", fmt=None, datefmt=None, style="%", **kwargs
):
"""Configure the file handler in charge of adding log messages to the heartbeat file
Parameters
----------
level: String, or Int, default='DEBUG'
Minimum message level for the heartbeat file. Passed to
:meth:`logging.FileHandler.setlevel`
fmt: String, or None, default=None
Message formatting string for the heartbeat file. Passed to
:meth:`logging.Formatter.__init__`
datefmt: String, or None, default=None
Date formatting string for the heartbeat file. Passed to
:meth:`logging.Formatter.__init__`
style: String, default='%'
Type of string formatting used. Passed to :meth:`logging.Formatter.__init__`
**kwargs: Dict
Extra keyword arguments
Returns
-------
file_handler: `logging.FileHandler` instance
The instantiated handler for the heartbeat file"""
if self.heartbeat_path is None:
raise FileExistsError
file_handler = logging.FileHandler(self.heartbeat_path, mode="w")
file_handler.setLevel(level)
fmt = fmt or "<%(asctime)s> %(levelname)-8s - %(message)s"
formatter = logging.Formatter(fmt=fmt, datefmt=datefmt, style=style)
file_handler.setFormatter(formatter)
return file_handler
##################################################
# Placeholder Methods:
##################################################
def log(self, content, **kwargs):
"""Placeholder method before proper initialization"""
def debug(self, content, **kwargs):
"""Placeholder method before proper initialization"""
def warn(self, content, **kwargs):
"""Placeholder method before proper initialization"""
##################################################
# Logging-Logging Methods:
##################################################
# noinspection PyUnusedLocal
def _logging_log(
self, content, verbose_threshold=None, previous_frame=None, add_time=False, **kwargs
):
"""Log an info message via the `logging` library
Parameters
----------
content: String
The message to log
verbose_threshold: Int, or None, default=None
If None, `content` logged normally. If int and `G.Env.verbose` >= `verbose_threshold`,
`content` is logged normally. Else if int and `G.Env.verbose` < `verbose_threshold`,
then `content` is logged on the `logging.debug` level, instead of `logging.info`
previous_frame: Frame, or None, default=None
The frame preceding the log call. If not provided, it will be inferred
add_time: Boolean, default=False
If True, the current time will be added to `content` before logging
**kwargs: Dict
Extra keyword arguments"""
if self.add_frame is True:
previous_frame = previous_frame or inspect.currentframe().f_back
try:
frame_source = format_frame_source(previous_frame)
finally:
del previous_frame
content = f"{frame_source} - {content}"
content = add_time_to_content(content, add_time=add_time)
if (verbose_threshold is None) or (G.Env.verbose >= verbose_threshold):
logging.info(content)
else:
logging.debug(content)
# noinspection PyUnusedLocal
def _logging_debug(self, content, previous_frame=None, add_time=False, **kwargs):
"""Log a debug message via the `logging` library
Parameters
----------
content: String
The message to log
previous_frame: Frame, or None, default=None
The frame preceding the debug call. If not provided, it will be inferred
add_time: Boolean, default=False
If True, the current time will be added to `content` before logging
**kwargs: Dict
Extra keyword arguments"""
if self.add_frame is True:
previous_frame = previous_frame or inspect.currentframe().f_back
try:
frame_source = format_frame_source(previous_frame)
finally:
del previous_frame
content = f"{frame_source} - {content}"
content = add_time_to_content(content, add_time=add_time)
logging.debug(content)
# noinspection PyUnusedLocal
def _logging_warn(self, content, **kwargs):
"""Log a warning message via the `logging` library
Parameters
----------
content: String
The message to log
**kwargs: Dict
Extra keyword arguments"""
if self.add_frame is True:
previous_frame = inspect.currentframe().f_back
try:
frame_source = format_frame_source(previous_frame)
finally:
del previous_frame
content = f"{frame_source} - {content}"
logging.warning(content)
class _Color:
"""Object defining color codes for use with logging"""
BLUE = "\033[34m"
CYAN = "\033[36m"
GREEN = "\033[32m"
MAGENTA = "\033[35m"
RED = "\033[31m"
STOP = "\033[0m"
def clean_parameter_names(parameter_names: list) -> List[str]:
"""Remove unnecessary prefixes or characters from the names of search space dimensions
Parameters
----------
parameter_names: List
Names of the dimensions in a hyperparameter search `Space` object. Values are usually tuples
Returns
-------
names: List[str]
Cleaned `parameter_names`, containing stringified values to facilitate logging"""
original_parameter_names = parameter_names.copy()
skip = ("model_init_params", "model_extra_params", "feature_engineer", "feature_selector")
names = [_[1:] if _[0] in skip else _ for _ in original_parameter_names]
names = [_[1:] if _[0] == "params" else _ for _ in names] # This is for Keras
names = [_[0] if len(_) == 1 else str(_).replace("'", "").replace('"', "") for _ in names]
# If a value in `names` is a 1-tuple, its single item is returned
# If a tuple with multiple items, the tuple is stringified, and quotation marks are removed
return names
def get_param_column_sizes(space: list, names: List[str]) -> List[int]:
"""Determine maximum column sizes for displaying values of each hyperparameter in `space`
Parameters
----------
space: List
Hyperparameter search space dimensions for the current Optimization Protocol
names: List[str]
Cleaned hyperparameter dimension names
Returns
-------
sizes: List[int]
Column sizes for each of the hyperparameters in `names`"""
sizes = [max(len(_), 7) for _ in names]
for i, dim in enumerate(space):
if isinstance(dim, Categorical):
str_categories = [getattr(_, "name", str(_)) for _ in dim.categories]
sizes[i] = max(sizes[i], *[len(_) for _ in str_categories])
return sizes
class OptimizationReporter:
def __init__(self, space: list, verbose=1, show_experiment_id=8, do_maximize=True):
"""A MixIn class for reporting the results of hyperparameter optimization rounds
Parameters
----------
space: List
Hyperparameter search space dimensions for the current Optimization Protocol
verbose: Int in [0, 1, 2], default=1
If 0, all but critical logging is silenced. If 1, normal logging is performed. If 2,
detailed logging is performed
show_experiment_id: Int, or Boolean, default=8
If True, the experiment_id will be printed in each result row. If False, it will not.
If int, the first `show_experiment_id`-many characters of each experiment_id will be
printed in each row
do_maximize: Boolean, default=True
If False, smaller metric values will be considered preferred and will be highlighted to
stand out. Else larger metric values will be treated as preferred"""
self.original_parameter_names = [_.name for _ in space]
self.verbose = verbose
self.show_experiment_id = (
36 if (show_experiment_id is True or show_experiment_id > 36) else show_experiment_id
)
self.do_maximize = do_maximize
self.end = "| "
self.y_max = None
self.x_max = None
self.iteration = 0
self.start_time = datetime.now()
self.last_round = datetime.now()
self.parameter_names = clean_parameter_names(self.original_parameter_names)
self.sizes = get_param_column_sizes(space, self.parameter_names)
self.sorted_indexes = sorted(
range(len(self.parameter_names)), key=self.parameter_names.__getitem__
)
def print_saved_results_header(self):
"""Print a header signifying that saved Experiment results are being read"""
header = f"{_Color.RED}Saved Results{_Color.STOP}"
self.print_header(header, (_Color.RED + "_" * self._line_len() + _Color.STOP))
def print_optimization_header(self):
"""Print a header signifying that Optimization rounds are starting"""
header = f"{_Color.RED}Hyperparameter Optimization{_Color.STOP}"
self.print_header(header, (_Color.RED + "_" * self._line_len() + _Color.STOP))
def _line_len(self):
"""Calculate number of characters a header's underlining should span
Returns
-------
line_len: Int
The number of characters the line should span"""
line_len = 24
# 24 is from " #| Time| Score|", which are the required heading columns, except
# for "ID", whose (optional) length is calculated below
# Can also be expressed as the sum of following four numbers:
# 5 == ``len(self.end) * 3 - 1`` (subtract 1 to drop extra space at right-side table end)
# 3 == size of "#" column
# 6 == size of "Time" column
# 10 == size of "Score" column
line_len += sum([_ + 4 for _ in self.sizes])
line_len += self.show_experiment_id + len(self.end) if self.show_experiment_id else 0
return line_len
def print_header(self, header, line):
"""Utility to perform actual printing of headers given formatted inputs
Parameters
----------
header: String
Specifies the stage of optimization being entered, and the type of results to follow
line: String
The underlining to follow `header`"""
print(header)
print(line)
self._print_column_name("#", 3)
if self.show_experiment_id:
self._print_column_name("ID", self.show_experiment_id)
self._print_column_name("Time", 6)
# size=6 because `expand_mins_secs` returns 2 units of time, each with 2 digits + 1 letter
self._print_column_name("Score", 10)
# size=10 for 5 fractional digits/mantissa, plus decimal itself, leaving 4 spaces for
# the integer part/characteristic, potentially prefixed by a negative symbol
for index in self.sorted_indexes:
self._print_column_name(self.parameter_names[index], self.sizes[index] + 2)
print("")
def _print_column_name(self, value, size):
"""Print a column name within a specified `size` constraint
Parameters
----------
value: String
The name of the column to print
size: Int
The number of characters that `value` should span"""
print("{0:>{1}}".format(value, size), end=self.end)
def print_result(self, hyperparameters, evaluation, experiment_id=None):
"""Print a row containing the results of an Experiment just executed
Parameters
----------
hyperparameters: List
List of hyperparameter values in the same order as :attr:`parameter_names`
evaluation: Float
An evaluation of the performance of `hyperparameters`
experiment_id: Str, or None, default=None
If not None, should be a string that is the UUID of the Experiment"""
if not self.verbose:
return
print("{:>3d}".format(self.iteration), end=self.end)
#################### Experiment ID ####################
if self.show_experiment_id:
if experiment_id is not None:
print("{}".format(experiment_id[: self.show_experiment_id]), end=self.end)
else:
print(" " * self.show_experiment_id, end=self.end)
#################### Time Elapsed ####################
minutes, seconds = divmod((datetime.now() - self.last_round).total_seconds(), 60)
print(expand_mins_secs(minutes, seconds), end=self.end)
#################### Evaluation Result ####################
if (
(self.y_max is None) # First evaluation
or (self.do_maximize and self.y_max < evaluation) # Found new max (best)
or (not self.do_maximize and self.y_max > evaluation) # Found new min (best)
):
self.y_max, self.x_max = evaluation, hyperparameters
self._print_target_value(evaluation, pre=_Color.MAGENTA, post=_Color.STOP)
self._print_input_values(hyperparameters, pre=_Color.GREEN, post=_Color.STOP)
else:
self._print_target_value(evaluation)
self._print_input_values(hyperparameters)
print("")
self.last_round = datetime.now()
self.iteration += 1
def _print_target_value(self, value, pre="", post=""):
"""Print the utility of an Experiment
Parameters
----------
value: String
The utility value to print
pre: String, default=''
Content to prepend to the formatted `value` string before printing
post: String, default=''
Content to append to the formatted `value` string before printing"""
content = pre + "{: >10.5f}".format(value) + post
print(content, end=self.end)
def _print_input_values(self, values, pre="", post=""):
"""Print the value of a hyperparameter used by an Experiment
Parameters
----------
value: String
The hyperparameter value to print
pre: String, default=''
Content to prepend to the formatted `value` string before printing
post: String, default=''
Content to append to the formatted `value` string before printing"""
for index in self.sorted_indexes:
if isinstance(values[index], float):
content = "{0: >{1}.{2}f}".format(
values[index], self.sizes[index] + 2, min(self.sizes[index] - 3, 6 - 2)
)
elif isinstance(values[index], tuple):
content = "{0!s: >{1}}".format(values[index], self.sizes[index] + 2)
# Above is nearly identical to below inside `try`, with addition of "!s". We're
# doing this only with tuples because otherwise it would print out a very
# verbose (`__repr__`) string for `EngineerStep`, rather than just its `name`
else:
try:
content = "{0: >{1}}".format(values[index], self.sizes[index] + 2)
except TypeError: # For `EngineerStep`
content = "{0: >{1}}".format(values[index].name, self.sizes[index] + 2)
print(pre + content + post, end=self.end)
def reset_timer(self):
"""Set :attr:`start_time`, and :attr:`last_round` to the current time"""
self.start_time = datetime.now()
self.last_round = datetime.now()
def print_summary(self):
"""Print a summary of the results of hyperparameter optimization upon completion"""
# TODO: Do this
def format_frame_source(previous_frame, **kwargs):
"""Construct a string describing the location at which a call was made
Parameters
----------
previous_frame: Frame
A frame depicting the location at which a call was made
**kwargs: Dict
Any additional kwargs to supply to :func:`reporting.stringify_frame_source`
Returns
-------
The stringified frame source information of `previous_frame`"""
source = inspect.getframeinfo(previous_frame)
src_script, src_line_no, src_func, src_class = source[0], source[1], source[2], None
with suppress(AttributeError, KeyError):
src_class = type(previous_frame.f_locals["self"]).__name__
return stringify_frame_source(src_script, src_line_no, src_func, src_class, **kwargs)
def stringify_frame_source(
src_file,
src_line_no,
src_func,
src_class,
add_line_no=True,
max_line_no_size=4,
total_max_size=80,
):
"""Construct a string that neatly displays the location in the code at which a call was made
Parameters
----------
src_file: Str
A filepath
src_line_no: Int
The line number in `src_file` at which the call was made
src_func: Str
The name of the function in `src_file` in which the call was made
src_class: Str, or None
If not None, the class in `src_file` in which the call was made
add_line_no: Boolean, default=False
If True, the line number will be included in the `source_content` result
max_line_no_size: Int, default=4
Total number (including padding) of characters to be occupied by `src_line_no`. For
example, if `src_line_no`=32, and `max_line_no_size`=4, `src_line_no` will be padded to
become '32 ' in order to occupy four characters
total_max_size: Int, default=80
Total number (including padding) of characters to be occupied by the `source_content` result
Returns
-------
source_content: Str
A formatted string containing the location in the code at which a call was made
Examples
--------
>>> stringify_frame_source("reporting.py", 570, "stringify_frame_source", None)
'570 - reporting.stringify_frame_source() '
>>> stringify_frame_source("reporting.py", 12, "bar", "Foo")
'12 - reporting.Foo.bar() '
>>> stringify_frame_source("reporting.py", 12, "bar", "Foo", add_line_no=False)
'reporting.Foo.bar() '
>>> stringify_frame_source("reporting.py", 12, "bar", "Foo", total_max_size=60)
'12 - reporting.Foo.bar() '"""
source_content = ""
if add_line_no is True:
# Left-align line_no to size: max_line_no_size
source_content += "{0:<{1}}".format(src_line_no, max_line_no_size)
source_content += " - "
script_name = os.path.splitext(os.path.basename(src_file))[0]
if src_class is not None:
source_content += "{}.{}.{}()".format(script_name, src_class, src_func)
else:
source_content += "{}.{}()".format(script_name, src_func)
source_content = "{0:<{1}}".format(source_content, total_max_size)
return source_content
def add_time_to_content(content, add_time=False):
"""Construct a string containing the original `content`, in addition to the current time
Parameters
----------
content: Str
The original string, to which the current time will be concatenated
add_time: Boolean, default=False
If True, the current time will be concatenated onto the end of `content`
Returns
-------
content: Str
Str containing original `content`, along with current time, and additional formatting"""
add_content = ""
add_time = now_time() if add_time is True else add_time
add_content += "Time: {}".format(add_time) if add_time else ""
#################### Combine Original and New Content ####################
if add_content != "":
content += " " if ((content != "") and (not content.endswith(" "))) else ""
content += add_content
return content
def format_fold_run(rep=None, fold=None, run=None, mode="concise"):
"""Construct a string to display the repetition, fold, and run currently being executed
Parameters
----------
rep: Int, or None, default=None
The repetition number currently being executed
fold: Int, or None, default=None
The fold number currently being executed
run: Int, or None, default=None
The run number currently being executed
mode: {"concise", "verbose"}, default="concise"
If "concise", the result will contain abbreviations for rep/fold/run
Returns
-------
content: Str
A clean display of the current repetition/fold/run
Examples
--------
>>> format_fold_run(rep=0, fold=3, run=2, mode="concise")
'R0-f3-r2'
>>> format_fold_run(rep=0, fold=3, run=2, mode="verbose")
'Rep-Fold-Run: 0-3-2'
>>> format_fold_run(rep=0, fold=3, run="*", mode="concise")
'R0-f3-r*'
>>> format_fold_run(rep=0, fold=3, run=2, mode="foo")
Traceback (most recent call last):
File "reporting.py", line ?, in format_fold_run
ValueError: Received invalid mode value: 'foo'"""
content = ""
if mode == "verbose":
content += format("Rep" if rep is not None else "")
content += format("-" if rep is not None and fold is not None else "")
content += format("Fold" if fold is not None else "")
content += format("-" if fold is not None and run is not None else "")
content += format("Run" if run is not None else "")
content += format(": " if any(_ is not None for _ in [rep, fold, run]) else "")
content += format(rep if rep is not None else "")
content += format("-" if rep is not None and fold is not None else "")
content += format(fold if fold is not None else "")
content += format("-" if fold is not None and run is not None else "")
content += format(run if run is not None else "")
elif mode == "concise":
content += format("R" if rep is not None else "")
content += format(rep if rep is not None else "")
content += format("-" if rep is not None and fold is not None else "")
content += format("f" if fold is not None else "")
content += format(fold if fold is not None else "")
content += format("-" if fold is not None and run is not None else "")
content += format("r" if run is not None else "")
content += format(run if run is not None else "")
else:
raise ValueError("Received invalid mode value: '{}'".format(mode))
return content
def format_evaluation(results, separator=" | ", float_format="{:.5f}"):
"""Construct a string to neatly display the results of a model evaluation
Parameters
----------
results: Dict
The results of a model evaluation, in which keys represent the dataset type evaluated, and
values are dicts containing metrics as keys, and metric values as values
separator: Str, default=' | '
The string used to join all the metric values into a single string
float_format: Str, default='{:.5f}'
A python string float formatter, applied to floating metric values
Returns
-------
content: Str
The model's evaluation results"""
content = []
for data_type, values in results.items():
if values is None:
continue
data_type = "OOF" if data_type == "oof" else data_type
data_type = "Holdout" if data_type == "holdout" else data_type
data_type = "In-Fold" if data_type == "in_fold" else data_type
metric_entry = "{}(".format(data_type)
metric_entry_vals = []
for metric_id, metric_value in values.items():
formatted_value = float_format.format(metric_value)
metric_entry_vals.append("{}={}".format(metric_id, formatted_value))
metric_entry += ", ".join(metric_entry_vals) + ")"
content.append(metric_entry)
content = separator.join(content)
return content
# ADVANCED_FIT_LOGGING_DISPLAY_LAYOUT = [
# {
# "column_name": "General",
# "sub_columns_names": [
# ["fold", "Fold"],
# ["run", "Run"],
# ["seed", "Seed"],
# ["step", "Step"],
# ["start_time", "Start Time"],
# ["end_time", "End Time"],
# ["time_elapsed", "Time Elapsed"]
# ],
# "sub_column_min_sizes": [10, 10, 10, 20, 12, 12, 12]
# },
# # Will need to alter default "Score" sub-columns according to what metrics are actually being used
# {
# "column_name": "OOF Scores",
# "sub_columns_names": [
# ["oof_f1", "F1"],
# ["oof_roc_auc", "ROC_AUC"]
# ]
# },
# # Check that Holdout dataset is in use before adding "Holdout Scores" column
# {
# "column_name": "Holdout Scores",
# "sub_columns_names": [
# ["holdout_f1", "F1"],
# ["holdout_roc_auc", "ROC_AUC"]
# ]
# },
# {
# "column_name": "Losses",
# "sub_columns_names": [
# ["train_loss", "Train"],
# ["validation_loss", "Validation"]
# ]
# },
# ]
#
#
# class AdvancedDisplayLayout(object):
# def __init__(self):
# pass
#
#
# class AdvancedFitLogging(object):
# def __init__(self, display_layout=None, ):
# self.display_layout = display_layout or ADVANCED_FIT_LOGGING_DISPLAY_LAYOUT
#
# def _validate_parameters(self):
# pass
#
# def validate_display_layout(self):
# pass