if (self.parameters.mode == base_layers.TRAIN and
        self.activation_dropout_rate > 0.0):
      tensor = tf.nn.dropout(tensor, rate=self.activation_dropout_rate)