TFMA Evaluators

tensorflow_model_analysis.evaluators

Init module for TensorFlow Model Analysis evaluators.

Attributes

Evaluation module-attribute

Evaluation = Dict[str, PCollection]

Functions

AnalysisTableEvaluator

AnalysisTableEvaluator(
    key: str = ANALYSIS_KEY,
    run_after: str = LAST_EXTRACTOR_STAGE_NAME,
    include: Optional[
        Union[Iterable[str], Dict[str, Any]]
    ] = None,
    exclude: Optional[
        Union[Iterable[str], Dict[str, Any]]
    ] = None,
) -> Evaluator

Creates an Evaluator for returning Extracts data for analysis.

If both include and exclude are None then tfma.INPUT_KEY extracts will be excluded by default.

PARAMETER	DESCRIPTION
key	Name to use for key in Evaluation output. TYPE: str DEFAULT: ANALYSIS_KEY
run_after	Extractor to run after (None means before any extractors). TYPE: str DEFAULT: LAST_EXTRACTOR_STAGE_NAME
include	List or map of keys to include in output. Keys starting with '_' are automatically filtered out at write time. If a map of keys is passed then the keys and sub-keys that exist in the map will be included in the output. An empty dict behaves as a wildcard matching all keys or the value itself. Since matching on feature values is not currently supported, an empty dict must be used to represent the leaf nodes. For example: {'key1': {'key1-subkey': {}}, 'key2': {}}. TYPE: Optional[Union[Iterable[str], Dict[str, Any]]] DEFAULT: None
exclude	List or map of keys to exclude from output. If a map of keys is passed then the keys and sub-keys that exist in the map will be excluded from the output. An empty dict behaves as a wildcard matching all keys or the value itself. Since matching on feature values is not currently supported, an empty dict must be used to represent the leaf nodes. For example, {'key1': {'key1-subkey': {}}, 'key2': {}}. TYPE: Optional[Union[Iterable[str], Dict[str, Any]]] DEFAULT: None

RETURNS	DESCRIPTION
Evaluator	Evaluator for collecting analysis data. The output is stored under the key
Evaluator	'analysis'.

RAISES	DESCRIPTION
ValueError	If both include and exclude are used.

Source code in tensorflow_model_analysis/evaluators/analysis_table_evaluator.py

def AnalysisTableEvaluator(  # pylint: disable=invalid-name
    key: str = constants.ANALYSIS_KEY,
    run_after: str = extractor.LAST_EXTRACTOR_STAGE_NAME,
    include: Optional[Union[Iterable[str], Dict[str, Any]]] = None,
    exclude: Optional[Union[Iterable[str], Dict[str, Any]]] = None,
) -> evaluator.Evaluator:
  """Creates an Evaluator for returning Extracts data for analysis.

  If both include and exclude are None then tfma.INPUT_KEY extracts will be
  excluded by default.

  Args:
    key: Name to use for key in Evaluation output.
    run_after: Extractor to run after (None means before any extractors).
    include: List or map of keys to include in output. Keys starting with '_'
      are automatically filtered out at write time. If a map of keys is passed
      then the keys and sub-keys that exist in the map will be included in the
      output. An empty dict behaves as a wildcard matching all keys or the value
      itself. Since matching on feature values is not currently supported, an
      empty dict must be used to represent the leaf nodes. For example: {'key1':
      {'key1-subkey': {}}, 'key2': {}}.
    exclude: List or map of keys to exclude from output. If a map of keys is
      passed then the keys and sub-keys that exist in the map will be excluded
      from the output. An empty dict behaves as a wildcard matching all keys or
      the value itself. Since matching on feature values is not currently
      supported, an empty dict must be used to represent the leaf nodes. For
      example, {'key1': {'key1-subkey': {}}, 'key2': {}}.

  Returns:
    Evaluator for collecting analysis data. The output is stored under the key
    'analysis'.

  Raises:
    ValueError: If both include and exclude are used.
  """
  # pylint: disable=no-value-for-parameter
  return evaluator.Evaluator(
      stage_name='EvaluateExtracts',
      run_after=run_after,
      ptransform=EvaluateExtracts(key=key, include=include, exclude=exclude),
  )

MetricsPlotsAndValidationsEvaluator

MetricsPlotsAndValidationsEvaluator(
    eval_config: EvalConfig,
    eval_shared_model: Optional[
        MaybeMultipleEvalSharedModels
    ] = None,
    metrics_key: str = METRICS_KEY,
    plots_key: str = PLOTS_KEY,
    attributions_key: str = ATTRIBUTIONS_KEY,
    run_after: str = SLICE_KEY_EXTRACTOR_STAGE_NAME,
    schema: Optional[Schema] = None,
    random_seed_for_testing: Optional[int] = None,
) -> Evaluator

Creates an Evaluator for evaluating metrics and plots.

PARAMETER	DESCRIPTION
eval_config	Eval config. TYPE: EvalConfig
eval_shared_model	Optional shared model (single-model evaluation) or list of shared models (multi-model evaluation). Only required if there are metrics to be computed in-graph using the model. TYPE: Optional[MaybeMultipleEvalSharedModels] DEFAULT: None
metrics_key	Name to use for metrics key in Evaluation output. TYPE: str DEFAULT: METRICS_KEY
plots_key	Name to use for plots key in Evaluation output. TYPE: str DEFAULT: PLOTS_KEY
attributions_key	Name to use for attributions key in Evaluation output. TYPE: str DEFAULT: ATTRIBUTIONS_KEY
run_after	Extractor to run after (None means before any extractors). TYPE: str DEFAULT: SLICE_KEY_EXTRACTOR_STAGE_NAME
schema	A schema to use for customizing metrics and plots. TYPE: Optional[Schema] DEFAULT: None
random_seed_for_testing	Seed to use for unit testing. TYPE: Optional[int] DEFAULT: None

RETURNS	DESCRIPTION
Evaluator	Evaluator for evaluating metrics and plots. The output will be stored under
Evaluator	'metrics' and 'plots' keys.

Source code in tensorflow_model_analysis/evaluators/metrics_plots_and_validations_evaluator.py

def MetricsPlotsAndValidationsEvaluator(  # pylint: disable=invalid-name
    eval_config: config_pb2.EvalConfig,
    eval_shared_model: Optional[types.MaybeMultipleEvalSharedModels] = None,
    metrics_key: str = constants.METRICS_KEY,
    plots_key: str = constants.PLOTS_KEY,
    attributions_key: str = constants.ATTRIBUTIONS_KEY,
    run_after: str = slice_key_extractor.SLICE_KEY_EXTRACTOR_STAGE_NAME,
    schema: Optional[schema_pb2.Schema] = None,
    random_seed_for_testing: Optional[int] = None,
) -> evaluator.Evaluator:
  """Creates an Evaluator for evaluating metrics and plots.

  Args:
    eval_config: Eval config.
    eval_shared_model: Optional shared model (single-model evaluation) or list
      of shared models (multi-model evaluation). Only required if there are
      metrics to be computed in-graph using the model.
    metrics_key: Name to use for metrics key in Evaluation output.
    plots_key: Name to use for plots key in Evaluation output.
    attributions_key: Name to use for attributions key in Evaluation output.
    run_after: Extractor to run after (None means before any extractors).
    schema: A schema to use for customizing metrics and plots.
    random_seed_for_testing: Seed to use for unit testing.

  Returns:
    Evaluator for evaluating metrics and plots. The output will be stored under
    'metrics' and 'plots' keys.
  """
  eval_shared_models = model_util.verify_and_update_eval_shared_models(
      eval_shared_model
  )
  if eval_shared_models:
    eval_shared_models = {m.model_name: m for m in eval_shared_models}

  # pylint: disable=no-value-for-parameter
  return evaluator.Evaluator(
      stage_name='EvaluateMetricsAndPlots',
      run_after=run_after,
      ptransform=_EvaluateMetricsPlotsAndValidations(
          eval_config=eval_config,
          eval_shared_models=eval_shared_models,
          metrics_key=metrics_key,
          plots_key=plots_key,
          attributions_key=attributions_key,
          schema=schema,
          random_seed_for_testing=random_seed_for_testing,
      ),
  )

verify_evaluator

verify_evaluator(
    evaluator: Evaluator, extractors: List[Extractor]
)

Verifies evaluator is matched with an extractor.

PARAMETER	DESCRIPTION
evaluator	Evaluator to verify. TYPE: Evaluator
extractors	Extractors to use in verification. TYPE: List[Extractor]

RAISES	DESCRIPTION
ValueError	If an Extractor cannot be found for the Evaluator.

Source code in tensorflow_model_analysis/evaluators/evaluator.py

def verify_evaluator(
    evaluator: Evaluator, extractors: List[extractor.Extractor]
):
  """Verifies evaluator is matched with an extractor.

  Args:
    evaluator: Evaluator to verify.
    extractors: Extractors to use in verification.

  Raises:
    ValueError: If an Extractor cannot be found for the Evaluator.
  """
  if (
      evaluator.run_after
      and evaluator.run_after != extractor.LAST_EXTRACTOR_STAGE_NAME
      and not any(evaluator.run_after == x.stage_name for x in extractors)
  ):
    raise ValueError(
        'Extractor matching run_after=%s for Evaluator %s not found'
        % (evaluator.run_after, evaluator.stage_name)
    )