video_level_models.py

# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Contains model definitions."""
import math

import models
import tensorflow as tf
import utils

from tensorflow import flags
import tensorflow.contrib.slim as slim

FLAGS = flags.FLAGS
flags.DEFINE_integer(
    "moe_num_mixtures", 2,
    "The number of mixtures (excluding the dummy 'expert') used for MoeModel.")


class LogisticModel(models.BaseModel):
    """Logistic model with L2 regularization."""

    def create_model(self, model_input, vocab_size=4716, l2_penalty=1e-8, **unused_params):
        """Creates a logistic model.

        Args:
          model_input: 'batch' x 'num_features' matrix of input features.
          vocab_size: The number of classes in the dataset.

        Returns:
          A dictionary with a tensor containing the probability predictions of the
          model in the 'predictions' key. The dimensions of the tensor are
          batch_size x num_classes."""
        output = slim.fully_connected(
            model_input, vocab_size, activation_fn=tf.nn.sigmoid,
            weights_regularizer=slim.l2_regularizer(l2_penalty))
        return {"predictions": output}


class EmbeddingModel(models.BaseModel):
    """Model from the paper with L2 regularization."""

    def create_model(self, model_input,
                    hid_1_audio=128,
                    hid_2_audio=128,
                    hid_1_frames=1024,
                    hid_2_frames=1024,
                    hid=128,
                    vocab_size=4716, l2_penalty=1e-8,
                    **unused_params):
        """Creates a model.

        Args:
          model_input: 'batch' x 'num_features' matrix of input features.
          vocab_size: The number of classes in the dataset.

        Returns:
          A dictionary with a tensor containing the probability predictions of the
          model in the 'predictions' key. The dimensions of the tensor are
          batch_size x num_classes."""

        model_input_audio = model_input[:, 1024:1024 + 128]
        model_input_frames = model_input[:, 0:1024]

        first_audio = slim.fully_connected(
            model_input_audio, hid_1_audio, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        second_audio = slim.fully_connected(
            first_audio, hid_1_audio, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        first_frames = slim.fully_connected(
            model_input_frames, hid_1_frames, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        second_frames = slim.fully_connected(
            first_frames, hid_1_frames, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        third_frames = slim.fully_connected(
            second_frames, hid_2_frames, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        third_audio = slim.fully_connected(
            second_audio, hid_2_audio, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        fourth_frames = slim.fully_connected(
            third_frames, hid_2_frames, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        fourth_audio = slim.fully_connected(
            third_audio, hid_2_audio, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        embedding_audio = slim.fully_connected(
            fourth_audio, hid, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        embedding_frames = slim.fully_connected(
            fourth_frames, hid, activation_fn=tf.nn.relu,
            weights_regularizer=slim.l2_regularizer(l2_penalty))

        embedding_frames = tf.nn.l2_normalize(embedding_frames, 1)
        embedding_audio = tf.nn.l2_normalize(embedding_audio, 1)

        embeddings = tf.concat([embedding_audio, embedding_frames], 1)

        # We use the same scope because we want the weights to be shared (to be the same)
        # between predictions_audio and predictions_frames
        predictions_audio = slim.fully_connected(
            embedding_audio, vocab_size, activation_fn=tf.nn.sigmoid,
            weights_regularizer=slim.l2_regularizer(l2_penalty), scope='predictions_layer')

        predictions_frames = slim.fully_connected(
            embedding_frames, vocab_size, activation_fn=tf.nn.sigmoid,
            weights_regularizer=slim.l2_regularizer(l2_penalty), scope='predictions_layer')

        predictions = tf.concat([predictions_audio, predictions_frames], 1)

        return {"predictions": predictions, "hidden_layer_activations": embeddings}

class MoeModel(models.BaseModel):
    """A softmax over a mixture of logistic models (with L2 regularization)."""

    def create_model(self,
                     model_input,
                     vocab_size,
                     num_mixtures=None,
                     l2_penalty=1e-8,
                     **unused_params):
        """Creates a Mixture of (Logistic) Experts model.

         The model consists of a per-class softmax distribution over a
         configurable number of logistic classifiers. One of the classifiers in the
         mixture is not trained, and always predicts 0.

        Args:
          model_input: 'batch_size' x 'num_features' matrix of input features.
          vocab_size: The number of classes in the dataset.
          num_mixtures: The number of mixtures (excluding a dummy 'expert' that
            always predicts the non-existence of an entity).
          l2_penalty: How much to penalize the squared magnitudes of parameter
            values.
        Returns:
          A dictionary with a tensor containing the probability predictions of the
          model in the 'predictions' key. The dimensions of the tensor are
          batch_size x num_classes.
        """
        num_mixtures = num_mixtures or FLAGS.moe_num_mixtures

        gate_activations = slim.fully_connected(
            model_input,
            vocab_size * (num_mixtures + 1),
            activation_fn=None,
            biases_initializer=None,
            weights_regularizer=slim.l2_regularizer(l2_penalty),
            scope="gates")
        expert_activations = slim.fully_connected(
            model_input,
            vocab_size * num_mixtures,
            activation_fn=None,
            weights_regularizer=slim.l2_regularizer(l2_penalty),
            scope="experts")

        gating_distribution = tf.nn.softmax(tf.reshape(
            gate_activations,
            [-1, num_mixtures + 1]))  # (Batch * #Labels) x (num_mixtures + 1)
        expert_distribution = tf.nn.sigmoid(tf.reshape(
            expert_activations,
            [-1, num_mixtures]))  # (Batch * #Labels) x num_mixtures

        final_probabilities_by_class_and_batch = tf.reduce_sum(
            gating_distribution[:, :num_mixtures] * expert_distribution, 1)
        final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
                                         [-1, vocab_size])
        return {"predictions": final_probabilities}