Skip to content

TFMA Evaluators

tensorflow_model_analysis.evaluators

Init module for TensorFlow Model Analysis evaluators.

Attributes

Evaluation module-attribute 

Evaluation = Dict[str, PCollection]

Classes

Functions

AnalysisTableEvaluator 

AnalysisTableEvaluator(
    key: str = ANALYSIS_KEY,
    run_after: str = LAST_EXTRACTOR_STAGE_NAME,
    include: Optional[
        Union[Iterable[str], Dict[str, Any]]
    ] = None,
    exclude: Optional[
        Union[Iterable[str], Dict[str, Any]]
    ] = None,
) -> Evaluator

Creates an Evaluator for returning Extracts data for analysis.

If both include and exclude are None then tfma.INPUT_KEY extracts will be excluded by default.

key: Name to use for key in Evaluation output. run_after: Extractor to run after (None means before any extractors). include: List or map of keys to include in output. Keys starting with '_' are automatically filtered out at write time. If a map of keys is passed then the keys and sub-keys that exist in the map will be included in the output. An empty dict behaves as a wildcard matching all keys or the value itself. Since matching on feature values is not currently supported, an empty dict must be used to represent the leaf nodes. For example: {'key1': {'key1-subkey': {}}, 'key2': {}}. exclude: List or map of keys to exclude from output. If a map of keys is passed then the keys and sub-keys that exist in the map will be excluded from the output. An empty dict behaves as a wildcard matching all keys or the value itself. Since matching on feature values is not currently supported, an empty dict must be used to represent the leaf nodes. For example, {'key1': {'key1-subkey': {}}, 'key2': {}}.

Evaluator for collecting analysis data. The output is stored under the key 'analysis'.

ValueError: If both include and exclude are used.

Source code in tensorflow_model_analysis/evaluators/analysis_table_evaluator.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
def AnalysisTableEvaluator(  # pylint: disable=invalid-name
    key: str = constants.ANALYSIS_KEY,
    run_after: str = extractor.LAST_EXTRACTOR_STAGE_NAME,
    include: Optional[Union[Iterable[str], Dict[str, Any]]] = None,
    exclude: Optional[Union[Iterable[str], Dict[str, Any]]] = None,
) -> evaluator.Evaluator:
    """Creates an Evaluator for returning Extracts data for analysis.

    If both include and exclude are None then tfma.INPUT_KEY extracts will be
    excluded by default.

    Args:
    ----
      key: Name to use for key in Evaluation output.
      run_after: Extractor to run after (None means before any extractors).
      include: List or map of keys to include in output. Keys starting with '_'
        are automatically filtered out at write time. If a map of keys is passed
        then the keys and sub-keys that exist in the map will be included in the
        output. An empty dict behaves as a wildcard matching all keys or the value
        itself. Since matching on feature values is not currently supported, an
        empty dict must be used to represent the leaf nodes. For example: {'key1':
        {'key1-subkey': {}}, 'key2': {}}.
      exclude: List or map of keys to exclude from output. If a map of keys is
        passed then the keys and sub-keys that exist in the map will be excluded
        from the output. An empty dict behaves as a wildcard matching all keys or
        the value itself. Since matching on feature values is not currently
        supported, an empty dict must be used to represent the leaf nodes. For
        example, {'key1': {'key1-subkey': {}}, 'key2': {}}.

    Returns:
    -------
      Evaluator for collecting analysis data. The output is stored under the key
      'analysis'.

    Raises:
    ------
      ValueError: If both include and exclude are used.
    """
    # pylint: disable=no-value-for-parameter
    return evaluator.Evaluator(
        stage_name="EvaluateExtracts",
        run_after=run_after,
        ptransform=EvaluateExtracts(key=key, include=include, exclude=exclude),
    )

MetricsPlotsAndValidationsEvaluator 

MetricsPlotsAndValidationsEvaluator(
    eval_config: EvalConfig,
    eval_shared_model: Optional[
        MaybeMultipleEvalSharedModels
    ] = None,
    metrics_key: str = METRICS_KEY,
    plots_key: str = PLOTS_KEY,
    attributions_key: str = ATTRIBUTIONS_KEY,
    run_after: str = SLICE_KEY_EXTRACTOR_STAGE_NAME,
    schema: Optional[Schema] = None,
    random_seed_for_testing: Optional[int] = None,
) -> Evaluator

Creates an Evaluator for evaluating metrics and plots.

eval_config: Eval config. eval_shared_model: Optional shared model (single-model evaluation) or list of shared models (multi-model evaluation). Only required if there are metrics to be computed in-graph using the model. metrics_key: Name to use for metrics key in Evaluation output. plots_key: Name to use for plots key in Evaluation output. attributions_key: Name to use for attributions key in Evaluation output. run_after: Extractor to run after (None means before any extractors). schema: A schema to use for customizing metrics and plots. random_seed_for_testing: Seed to use for unit testing.

Evaluator for evaluating metrics and plots. The output will be stored under 'metrics' and 'plots' keys.

Source code in tensorflow_model_analysis/evaluators/metrics_plots_and_validations_evaluator.py 
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def MetricsPlotsAndValidationsEvaluator(  # pylint: disable=invalid-name
    eval_config: config_pb2.EvalConfig,
    eval_shared_model: Optional[types.MaybeMultipleEvalSharedModels] = None,
    metrics_key: str = constants.METRICS_KEY,
    plots_key: str = constants.PLOTS_KEY,
    attributions_key: str = constants.ATTRIBUTIONS_KEY,
    run_after: str = slice_key_extractor.SLICE_KEY_EXTRACTOR_STAGE_NAME,
    schema: Optional[schema_pb2.Schema] = None,
    random_seed_for_testing: Optional[int] = None,
) -> evaluator.Evaluator:
    """Creates an Evaluator for evaluating metrics and plots.

    Args:
    ----
      eval_config: Eval config.
      eval_shared_model: Optional shared model (single-model evaluation) or list
        of shared models (multi-model evaluation). Only required if there are
        metrics to be computed in-graph using the model.
      metrics_key: Name to use for metrics key in Evaluation output.
      plots_key: Name to use for plots key in Evaluation output.
      attributions_key: Name to use for attributions key in Evaluation output.
      run_after: Extractor to run after (None means before any extractors).
      schema: A schema to use for customizing metrics and plots.
      random_seed_for_testing: Seed to use for unit testing.

    Returns:
    -------
      Evaluator for evaluating metrics and plots. The output will be stored under
      'metrics' and 'plots' keys.
    """
    eval_shared_models = model_util.verify_and_update_eval_shared_models(
        eval_shared_model
    )
    if eval_shared_models:
        eval_shared_models = {m.model_name: m for m in eval_shared_models}

    # pylint: disable=no-value-for-parameter
    return evaluator.Evaluator(
        stage_name="EvaluateMetricsAndPlots",
        run_after=run_after,
        ptransform=_EvaluateMetricsPlotsAndValidations(
            eval_config=eval_config,
            eval_shared_models=eval_shared_models,
            metrics_key=metrics_key,
            plots_key=plots_key,
            attributions_key=attributions_key,
            schema=schema,
            random_seed_for_testing=random_seed_for_testing,
        ),
    )

verify_evaluator 

verify_evaluator(
    evaluator: Evaluator, extractors: List[Extractor]
)

Verifies evaluator is matched with an extractor.

evaluator: Evaluator to verify. extractors: Extractors to use in verification.

ValueError: If an Extractor cannot be found for the Evaluator.

Source code in tensorflow_model_analysis/evaluators/evaluator.py 
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
def verify_evaluator(evaluator: Evaluator, extractors: List[extractor.Extractor]):
    """Verifies evaluator is matched with an extractor.

    Args:
    ----
      evaluator: Evaluator to verify.
      extractors: Extractors to use in verification.

    Raises:
    ------
      ValueError: If an Extractor cannot be found for the Evaluator.
    """
    if (
        evaluator.run_after
        and evaluator.run_after != extractor.LAST_EXTRACTOR_STAGE_NAME
        and not any(evaluator.run_after == x.stage_name for x in extractors)
    ):
        raise ValueError(
            "Extractor matching run_after=%s for Evaluator %s not found"
            % (evaluator.run_after, evaluator.stage_name)
        )