if (self.parameters.mode == base_layers.TRAIN and
        self.activation_dropout_rate > 0.0):
      ffn_down = tf.nn.dropout(ffn_down, rate=self.activation_dropout_rate)