TpuYouTube.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TpuYouTube.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "cChdgXab3M3R"
      },
      "source": [
        "import tensorflow as tf\r\n",
        "import os\r\n",
        "import numpy as np\r\n",
        "import tensorflow_datasets as tfds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GuK8JKS3EZMc"
      },
      "source": [
        "# Check if TPU is set as hardware accelerator and get tpu address"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xrqN2t8s49Db"
      },
      "source": [
        "assert 'COLAB_TPU_ADDR' in os.environ, 'Missing TPU; did you request a TPU in Notebook Settings?'\r\n",
        "\r\n",
        "if 'COLAB_TPU_ADDR' in os.environ:\r\n",
        "  TF_MASTER = 'grpc://{}'.format(os.environ['COLAB_TPU_ADDR'])\r\n",
        "else:\r\n",
        "  TF_MASTER=''\r\n",
        "\r\n",
        "tpu_address = TF_MASTER"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sPgy938hEzUI"
      },
      "source": [
        "# Initialize the TPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ec4IGavd5GN1",
        "outputId": "30af6d17-6dac-4b31-e2bb-9fb275412521"
      },
      "source": [
        "resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu_address)\r\n",
        "tf.config.experimental_connect_to_cluster(resolver)\r\n",
        "# This is the TPU initialization code that has to be at the beginning.\r\n",
        "tf.tpu.experimental.initialize_tpu_system(resolver)\r\n",
        "print(\"All devices: \", tf.config.list_logical_devices('TPU'))\r\n",
        "print(\"Number of devices: \", len(tf.config.list_logical_devices('TPU')))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.108.236.34:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.108.236.34:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "All devices:  [LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:7', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:6', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:5', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:4', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:3', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:0', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:1', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:2', device_type='TPU')]\n",
            "Number of devices:  8\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ck2fwEQkFAAC"
      },
      "source": [
        "# Choose a distribution strategy for the hardware accelerator\r\n",
        "# - Run the model on multiple TPUs in a data parallel way"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5V4Lq2sYFRK9",
        "outputId": "513c3508-9e76-4ed4-e892-4e58f2e4083b"
      },
      "source": [
        "strategy = tf.distribute.TPUStrategy(resolver)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k7irzi6FFptb"
      },
      "source": [
        "# Create a Convolutional Neural Network and train it on TPUs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WOrwXdkOFkJl"
      },
      "source": [
        "def create_model():\r\n",
        "  return tf.keras.Sequential(\r\n",
        "      [tf.keras.layers.Conv2D(256, (3,3), activation='relu', input_shape=(28, 28, 1)),\r\n",
        "       tf.keras.layers.Conv2D(256, (3,3), activation='relu'),\r\n",
        "       tf.keras.layers.Flatten(),\r\n",
        "       tf.keras.layers.Dense(256, activation='relu'),\r\n",
        "       tf.keras.layers.Dense(128, activation='relu'),\r\n",
        "       tf.keras.layers.Dense(10)\r\n",
        "       ])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TBvY5NG5OLFm"
      },
      "source": [
        "def create_model():\r\n",
        "  return tf.keras.Sequential(\r\n",
        "      [tf.keras.layers.Conv2D(32, (5,5), activation='relu', padding='same', input_shape=(28, 28, 1)),\r\n",
        "       tf.keras.layers.MaxPooling2D(pool_size=(2,2), padding='same'),\r\n",
        "       tf.keras.layers.Conv2D(32, (5,5), padding='same', activation='relu'),\r\n",
        "       tf.keras.layers.MaxPooling2D(pool_size=(2,2), padding='same'),\r\n",
        "       tf.keras.layers.Flatten(),\r\n",
        "       tf.keras.layers.Dense(1024, activation='relu'),\r\n",
        "       tf.keras.layers.Dropout(0.4),\r\n",
        "       tf.keras.layers.Dense(10)\r\n",
        "       ])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2OEtrK4_GAAq"
      },
      "source": [
        "# Load in the MNIST Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I5bh_AjBJjSR"
      },
      "source": [
        "def get_dataset(batch_size, is_training=True):\r\n",
        "  split = 'train' if is_training else 'test'\r\n",
        "  dataset, info = tfds.load(name='mnist', split=split, with_info=True,\r\n",
        "                            as_supervised=True, try_gcs=True)\r\n",
        "\r\n",
        "  def scale(image, label):\r\n",
        "    image = tf.cast(image, tf.float32)\r\n",
        "    image /= 255.0\r\n",
        "\r\n",
        "    return image, label\r\n",
        "\r\n",
        "  dataset = dataset.map(scale)\r\n",
        "\r\n",
        "  # Only shuffle and repeat the dataset in training. The advantage to have a\r\n",
        "  # infinite dataset for training is to avoid the potential last partial batch\r\n",
        "  # in each epoch, so users don't need to think about scaling the gradients\r\n",
        "  # based on the actual batch size.\r\n",
        "  if is_training:\r\n",
        "    dataset = dataset.shuffle(10000)\r\n",
        "    dataset = dataset.repeat()\r\n",
        "\r\n",
        "  dataset = dataset.batch(batch_size)\r\n",
        "\r\n",
        "  return dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5k45amayKmpZ"
      },
      "source": [
        "# Compile the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r7SahzElJmjV",
        "outputId": "bed1cc25-491c-46f4-e906-ae9c288d0d40"
      },
      "source": [
        "with strategy.scope():\r\n",
        "  model = create_model()\r\n",
        "  model.compile(optimizer='adam',\r\n",
        "                loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\r\n",
        "                metrics=['sparse_categorical_accuracy'])\r\n",
        "  \r\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 26, 26, 256)       2560      \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 24, 24, 256)       590080    \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 147456)            0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 256)               37748992  \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 128)               32896     \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 10)                1290      \n",
            "=================================================================\n",
            "Total params: 38,375,818\n",
            "Trainable params: 38,375,818\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S7EAvCgjG3om"
      },
      "source": [
        "# Train the model on TPUs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ffmM27sDKleb"
      },
      "source": [
        "batch_size = 200\r\n",
        "\r\n",
        "train_dataset = get_dataset(batch_size, is_training=True)\r\n",
        "test_dataset = get_dataset(batch_size, is_training=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u5h5SgAsKvdi",
        "outputId": "d61e7b79-50a5-41b9-da11-f98f422f4d82"
      },
      "source": [
        "epochs = 5\r\n",
        "steps_per_epoch = 60000 // batch_size\r\n",
        "validation_steps = 10000 // batch_size\r\n",
        "\r\n",
        "history = model.fit(train_dataset,\r\n",
        "                    epochs=epochs,\r\n",
        "                    steps_per_epoch=steps_per_epoch,\r\n",
        "                    validation_data=test_dataset, \r\n",
        "                    validation_steps=validation_steps)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "300/300 [==============================] - 21s 44ms/step - loss: 0.3620 - sparse_categorical_accuracy: 0.8913 - val_loss: 0.0422 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 2/5\n",
            "300/300 [==============================] - 10s 34ms/step - loss: 0.0389 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0485 - val_sparse_categorical_accuracy: 0.9847\n",
            "Epoch 3/5\n",
            "300/300 [==============================] - 10s 35ms/step - loss: 0.0232 - sparse_categorical_accuracy: 0.9929 - val_loss: 0.0520 - val_sparse_categorical_accuracy: 0.9850\n",
            "Epoch 4/5\n",
            "300/300 [==============================] - 11s 38ms/step - loss: 0.0151 - sparse_categorical_accuracy: 0.9951 - val_loss: 0.0367 - val_sparse_categorical_accuracy: 0.9901\n",
            "Epoch 5/5\n",
            "300/300 [==============================] - 11s 36ms/step - loss: 0.0079 - sparse_categorical_accuracy: 0.9977 - val_loss: 0.0511 - val_sparse_categorical_accuracy: 0.9870\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "89Z0JlZDHxD7",
        "outputId": "da4dadd9-0d7f-4fd1-c9a5-2a12bb93e1aa"
      },
      "source": [
        "with strategy.scope():\r\n",
        "  model = create_model()\r\n",
        "  model.compile(optimizer='adam',\r\n",
        "                # Anything between 2 and `steps_per_epoch` could help here.\r\n",
        "                steps_per_execution = 50,\r\n",
        "                loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\r\n",
        "                metrics=['sparse_categorical_accuracy'])\r\n",
        "\r\n",
        "epochs = 5\r\n",
        "steps_per_epoch = 60000 // batch_size\r\n",
        "validation_steps = 10000 // batch_size\r\n",
        "\r\n",
        "history = model.fit(train_dataset,\r\n",
        "                epochs=5,\r\n",
        "                steps_per_epoch=steps_per_epoch,\r\n",
        "                validation_data=test_dataset,\r\n",
        "                validation_steps=validation_steps)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "300/300 [==============================] - 15s 51ms/step - loss: 0.2203 - sparse_categorical_accuracy: 0.9332 - val_loss: 0.0466 - val_sparse_categorical_accuracy: 0.9853\n",
            "Epoch 2/5\n",
            "300/300 [==============================] - 5s 18ms/step - loss: 0.0359 - sparse_categorical_accuracy: 0.9889 - val_loss: 0.0442 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 3/5\n",
            "300/300 [==============================] - 5s 18ms/step - loss: 0.0193 - sparse_categorical_accuracy: 0.9940 - val_loss: 0.0395 - val_sparse_categorical_accuracy: 0.9885\n",
            "Epoch 4/5\n",
            "300/300 [==============================] - 6s 18ms/step - loss: 0.0141 - sparse_categorical_accuracy: 0.9953 - val_loss: 0.0527 - val_sparse_categorical_accuracy: 0.9847\n",
            "Epoch 5/5\n",
            "300/300 [==============================] - 5s 18ms/step - loss: 0.0093 - sparse_categorical_accuracy: 0.9970 - val_loss: 0.0530 - val_sparse_categorical_accuracy: 0.9870\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JQWpuOLzK7Rc"
      },
      "source": [
        "# Visualization of Training Process"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RZpMDXrOK65y"
      },
      "source": [
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jCnLq85GLA0X"
      },
      "source": [
        "# Plot the accuracy of the model during training\r\n",
        "plt.plot(history.history['sparse_categorical_accuracy'])\r\n",
        "plt.plot(history.history['val_sparse_categorical_accuracy'])\r\n",
        "plt.title('Model Accuracy')\r\n",
        "plt.ylabel('Sparse_categorical_accuracy')\r\n",
        "plt.xlabel('Epoch')\r\n",
        "plt.legend(['Training', 'Validation'], loc='upper left')\r\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_J3sua_NGHu"
      },
      "source": [
        "# Plot the loss of the model during training\r\n",
        "plt.plot(history.history['loss'])\r\n",
        "plt.plot(history.history['val_sparse_categorical_accuracy'])\r\n",
        "plt.title('Model Loss')\r\n",
        "plt.ylabel('Loss')\r\n",
        "plt.xlabel('Epoch')\r\n",
        "plt.legend(['Training', 'Validation'], loc='upper right')\r\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b4KqJMMIQy9n"
      },
      "source": [
        "# Evaluate the best model with testing data.\r\n",
        " evaluation = model.evaluate(test_dataset)\r\n",
        " print(evaluation)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}