Copyright 2020 The TensorFlow Authors.¶

In [ ]:

#@title Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#@title Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.

Fairness Indicators TensorBoard Plugin Example Colab¶

View on TensorFlow.org

Run in Google Colab

View source on GitHub

Download notebook

Overview¶

In this activity, you'll use Fairness Indicators for TensorBoard. With the plugin, you can visualize fairness evaluations for your runs and easily compare performance across groups.

Importing¶

Run the following code to install the required libraries.

In [ ]:

!pip install -q -U pip==20.2

!pip install fairness_indicators 'absl-py<0.9,>=0.7'
!pip install google-api-python-client==1.8.3
!pip install tensorboard-plugin-fairness-indicators
!pip install tensorflow-serving-api==2.16.1
!pip install tensorflow-model-analysis

!pip install -q -U pip==20.2 !pip install fairness_indicators 'absl-py<0.9,>=0.7' !pip install google-api-python-client==1.8.3 !pip install tensorboard-plugin-fairness-indicators !pip install tensorflow-serving-api==2.16.1 !pip install tensorflow-model-analysis

Restart the runtime. After the runtime is restarted, continue with following cells without running previous cell again.

In [ ]:

# %tf.disable_v2_behavior()	# Uncomment this line if running in Google Colab.

# %tf.disable_v2_behavior() # Uncomment this line if running in Google Colab.

In [ ]:

import datetime
import os
import tempfile
from tensorboard_plugin_fairness_indicators import summary_v2
import tensorflow.compat.v1 as tf
import numpy as np
from tensorflow import keras
from google.protobuf import text_format

# example_model.py is provided in fairness_indicators package to train and
# evaluate an example model.
from fairness_indicators import example_model
import tensorflow_model_analysis as tfma

tf.compat.v1.enable_eager_execution()

import datetime import os import tempfile from tensorboard_plugin_fairness_indicators import summary_v2 import tensorflow.compat.v1 as tf import numpy as np from tensorflow import keras from google.protobuf import text_format # example_model.py is provided in fairness_indicators package to train and # evaluate an example model. from fairness_indicators import example_model import tensorflow_model_analysis as tfma tf.compat.v1.enable_eager_execution()

Data and Constants¶

In [ ]:

# To know about dataset, check Fairness Indicators Example Colab at:
# https://github.com/tensorflow/fairness-indicators/blob/master/g3doc/tutorials/Fairness_Indicators_Example_Colab.ipynb

train_tf_file = tf.keras.utils.get_file('train.tf', 'https://storage.googleapis.com/civil_comments_dataset/train_tf_processed.tfrecord')
validate_tf_file = tf.keras.utils.get_file('validate.tf', 'https://storage.googleapis.com/civil_comments_dataset/validate_tf_processed.tfrecord')

BASE_DIR = tempfile.gettempdir()
TEXT_FEATURE = 'comment_text'
LABEL = 'toxicity'
FEATURE_MAP = {
    # Label:
    LABEL: tf.io.FixedLenFeature([], tf.float32),
    # Text:
    TEXT_FEATURE: tf.io.FixedLenFeature([], tf.string),

    # Identities:
    'sexual_orientation': tf.io.VarLenFeature(tf.string),
    'gender': tf.io.VarLenFeature(tf.string),
    'religion': tf.io.VarLenFeature(tf.string),
    'race': tf.io.VarLenFeature(tf.string),
    'disability': tf.io.VarLenFeature(tf.string),
}

# To know about dataset, check Fairness Indicators Example Colab at: # https://github.com/tensorflow/fairness-indicators/blob/master/g3doc/tutorials/Fairness_Indicators_Example_Colab.ipynb train_tf_file = tf.keras.utils.get_file('train.tf', 'https://storage.googleapis.com/civil_comments_dataset/train_tf_processed.tfrecord') validate_tf_file = tf.keras.utils.get_file('validate.tf', 'https://storage.googleapis.com/civil_comments_dataset/validate_tf_processed.tfrecord') BASE_DIR = tempfile.gettempdir() TEXT_FEATURE = 'comment_text' LABEL = 'toxicity' FEATURE_MAP = { # Label: LABEL: tf.io.FixedLenFeature([], tf.float32), # Text: TEXT_FEATURE: tf.io.FixedLenFeature([], tf.string), # Identities: 'sexual_orientation': tf.io.VarLenFeature(tf.string), 'gender': tf.io.VarLenFeature(tf.string), 'religion': tf.io.VarLenFeature(tf.string), 'race': tf.io.VarLenFeature(tf.string), 'disability': tf.io.VarLenFeature(tf.string), }

Train the Model¶

In [ ]:

model_dir = os.path.join(BASE_DIR, 'train',
                         datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))

model_dir = os.path.join(BASE_DIR, 'train', datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))

In [ ]:

classifier = example_model.get_example_model(example_model.TEXT_FEATURE)
classifier.compile(optimizer=keras.optimizers.Adam(), loss='mse')

# Read the data from the training file
data = []
dataset = tf.data.Dataset.list_files(train_tf_file, shuffle=False)
dataset = dataset.flat_map(tf.data.TFRecordDataset)
for raw_record in dataset.take(1):
  example = tf.train.Example()
  example.ParseFromString(raw_record.numpy())
  data.append(example)

classifier.fit(
    tf.constant([e.SerializeToString() for e in data]),
    np.array([
        e.features.feature[example_model.LABEL].float_list.value[:][0]
        for e in data
    ]),
)
classifier.save(model_dir, save_format='tf')

classifier = example_model.get_example_model(example_model.TEXT_FEATURE) classifier.compile(optimizer=keras.optimizers.Adam(), loss='mse') # Read the data from the training file data = [] dataset = tf.data.Dataset.list_files(train_tf_file, shuffle=False) dataset = dataset.flat_map(tf.data.TFRecordDataset) for raw_record in dataset.take(1): example = tf.train.Example() example.ParseFromString(raw_record.numpy()) data.append(example) classifier.fit( tf.constant([e.SerializeToString() for e in data]), np.array([ e.features.feature[example_model.LABEL].float_list.value[:][0] for e in data ]), ) classifier.save(model_dir, save_format='tf')

Run TensorFlow Model Analysis with Fairness Indicators¶

This step might take 2 to 5 minutes.

In [ ]:

tfma_eval_result_path = os.path.join(BASE_DIR, 'tfma_eval_result')

eval_config = text_format.Parse(
    """
    model_specs {
      signature_name: "serving_default"
      prediction_key: "predictions" # placeholder
      label_key: "toxicity" # placeholder
    }
    slicing_specs {}
    slicing_specs {
      feature_keys: ["gender"]
    }
    metrics_specs {
      metrics {
        class_name: "ExampleCount"
      }
      metrics {
        class_name: "FairnessIndicators"
      }
    }
""",
    tfma.EvalConfig(),
)

tfma_eval_result_path = os.path.join(model_dir, 'tfma_eval_result')
example_model.evaluate_model(
    model_dir,
    validate_tf_file,
    tfma_eval_result_path,
    eval_config,
)

tfma_eval_result_path = os.path.join(BASE_DIR, 'tfma_eval_result') eval_config = text_format.Parse( """ model_specs { signature_name: "serving_default" prediction_key: "predictions" # placeholder label_key: "toxicity" # placeholder } slicing_specs {} slicing_specs { feature_keys: ["gender"] } metrics_specs { metrics { class_name: "ExampleCount" } metrics { class_name: "FairnessIndicators" } } """, tfma.EvalConfig(), ) tfma_eval_result_path = os.path.join(model_dir, 'tfma_eval_result') example_model.evaluate_model( model_dir, validate_tf_file, tfma_eval_result_path, eval_config, )

Visualize Fairness Indicators in TensorBoard¶

Below you will visualize Fairness Indicators in Tensorboard and compare performance of each slice of the data on selected metrics. You can adjust the baseline comparison slice as well as the displayed threshold(s) using the drop down menus at the top of the visualization. You can also select different evaluation runs using the drop down menu at the top-left corner.

Write Fairness Indicators Summary¶

Write summary file containing all required information to visualize Fairness Indicators in TensorBoard.

In [ ]:

import tensorflow.compat.v2 as tf2

writer = tf2.summary.create_file_writer(
    os.path.join(model_dir, 'fairness_indicators'))
with writer.as_default():
  summary_v2.FairnessIndicators(tfma_eval_result_path, step=1)
writer.close()

import tensorflow.compat.v2 as tf2 writer = tf2.summary.create_file_writer( os.path.join(model_dir, 'fairness_indicators')) with writer.as_default(): summary_v2.FairnessIndicators(tfma_eval_result_path, step=1) writer.close()

Launch TensorBoard¶

Navigate to "Fairness Indicators" tab to visualize Fairness Indicators.

In [ ]:

%load_ext tensorboard

%load_ext tensorboard

In [ ]:

%tensorboard --logdir=$model_dir

%tensorboard --logdir=$model_dir