EmoInt_Emotion_Analysis_Bidirectional_LSTM_Threshold_0_33.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/nakamura41/emotion_detection_notebooks/blob/master/EmoInt_Emotion_Analysis_Bidirectional_LSTM_Threshold_0_33.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "colab_type": "code",
    "id": "tgVlWx0J5in1",
    "outputId": "c3feab17-008f-4d97-950a-f90fcf3f217d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: PyDrive in /usr/local/lib/python3.6/dist-packages (1.3.1)\n",
      "Requirement already satisfied: oauth2client>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (4.1.3)\n",
      "Requirement already satisfied: PyYAML>=3.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (3.13)\n",
      "Requirement already satisfied: google-api-python-client>=1.2 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (1.6.7)\n",
      "Requirement already satisfied: httplib2>=0.9.1 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.11.3)\n",
      "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (4.0)\n",
      "Requirement already satisfied: six>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (1.11.0)\n",
      "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.4.4)\n",
      "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.2.2)\n",
      "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (3.0.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install PyDrive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "id": "OdaHCMv2rGKs",
    "outputId": "757ce36e-5dc9-4f0b-888a-afe86c9e8511"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import numpy as np\n",
    "import urllib.request\n",
    "import pandas as pd\n",
    "from gensim.test.utils import common_texts, get_tmpfile\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.models import Word2Vec\n",
    "from tempfile import gettempdir\n",
    "import zipfile\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from pydrive.auth import GoogleAuth\n",
    "from pydrive.drive import GoogleDrive\n",
    "from google.colab import auth\n",
    "from oauth2client.client import GoogleCredentials\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, precision_recall_fscore_support, accuracy_score\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VMhX1N2k4SgY"
   },
   "outputs": [],
   "source": [
    "# 1. Authenticate and create the PyDrive client.\n",
    "auth.authenticate_user()\n",
    "gauth = GoogleAuth()\n",
    "gauth.credentials = GoogleCredentials.get_application_default()\n",
    "drive = GoogleDrive(gauth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6FJlfk614hrw"
   },
   "outputs": [],
   "source": [
    "# download google drive file\n",
    "def download_drive_file(drive_directory, filename):\n",
    "  list_file_query = \"title='{}' and trashed=false\".format(drive_directory)\n",
    "  file_list = drive.ListFile({'q': list_file_query}).GetList()\n",
    "\n",
    "  if len(file_list) > 0:\n",
    "    directory_id = file_list[0]['id']\n",
    "\n",
    "    list_file_query = \"'{}' in parents\".format(directory_id)\n",
    "\n",
    "    file_list = drive.ListFile({'q': list_file_query}).GetList()\n",
    "    \n",
    "    file_id = None\n",
    "    for file1 in file_list:\n",
    "      if file1['title'] == filename:\n",
    "        print(\"downloading file {}, file id {}\".format(file1['title'], file1['id']))\n",
    "        file1.GetContentFile(file1['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VRAY8D9JtRHk"
   },
   "outputs": [],
   "source": [
    "# upload google drive file\n",
    "def upload_drive_file(drive_directory, filename):\n",
    "  list_file_query = \"title='{}' and trashed=false\".format(drive_directory)\n",
    "  file_list = drive.ListFile({'q': list_file_query}).GetList()\n",
    "\n",
    "  if len(file_list) > 0:\n",
    "    directory_id = file_list[0]['id']\n",
    "    file = drive.CreateFile({\n",
    "        'title': filename, \n",
    "        \"parents\": [{\"kind\": \"drive#fileLink\",\"id\": directory_id}]\n",
    "    })\n",
    "    file.SetContentFile(filename)\n",
    "    file.Upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cWPQ7vL6uFNK"
   },
   "outputs": [],
   "source": [
    "def save_numpy_array(numpy_array, filename):\n",
    "  np.save(filename, numpy_array)\n",
    "  \n",
    "def save_n_upload_numpy_array(numpy_array, drive_directory, filename):\n",
    "  save_numpy_array(numpy_array, filename)\n",
    "  upload_drive_file(drive_directory, filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oBynQmEcrNAy"
   },
   "outputs": [],
   "source": [
    "class EmoIntDataset(object):\n",
    "  BASE_URL = \"http://saifmohammad.com/WebDocs/\"\n",
    "  TRAIN_URI = \"EmoInt%20Train%20Data/{}-ratings-0to1.train.txt\"\n",
    "  TEST_URI = \"EmoInt%20Test%20Gold%20Data/{}-ratings-0to1.test.gold.txt\"\n",
    "  EMOTION_CLASSES = [\"anger\", \"fear\", \"joy\", \"sadness\"]\n",
    "  \n",
    "  THRESHOLD = 0.33\n",
    "  \n",
    "  def __load_data_per_class(self, url, threshold=0):\n",
    "    resource = urllib.request.urlopen(url)\n",
    "    np_array = np.asarray([line.split('\\t') for line in [line.strip() for line in resource.read().decode('utf-8').splitlines()]])\n",
    "    df = pd.DataFrame(np_array, columns=[\"id\", \"text\", \"emotion\", \"emotion_level\"])\n",
    "    df['emotion_level'] = df['emotion_level'].astype(float)\n",
    "    df = df.query('emotion_level>' + str(threshold))\n",
    "    return df[[\"text\", \"emotion\"]]\n",
    "  \n",
    "  def load_data(self, set_threshold=False):\n",
    "    train_data = None\n",
    "    test_data = None\n",
    "    \n",
    "    for emotion in self.EMOTION_CLASSES:\n",
    "      # load train dataset\n",
    "      train_df = self.__load_data_per_class(self.BASE_URL + self.TRAIN_URI.format(emotion), threshold=(self.THRESHOLD if set_threshold else 0))\n",
    "      \n",
    "      # load test dataset\n",
    "      test_df = self.__load_data_per_class(self.BASE_URL + self.TEST_URI.format(emotion), threshold=(self.THRESHOLD if set_threshold else 0))\n",
    "      \n",
    "      train_data = (train_df if train_data is None else train_data.append(train_df))\n",
    "      test_data = (test_df if test_data is None else test_data.append(test_df))\n",
    "      \n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "tevwfq4Tu6fe",
    "outputId": "5507d39f-e094-4727-d761-2c4d1a883ecb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2610, 2)\n",
      "(291, 2)\n",
      "(2508, 2)\n"
     ]
    }
   ],
   "source": [
    "emo_int_dataset = EmoIntDataset()\n",
    "train_data, test_data = emo_int_dataset.load_data(set_threshold=True)\n",
    "train_data, valid_data = train_test_split(train_data, test_size=0.1, random_state=200, stratify=train_data.emotion)\n",
    "\n",
    "print(train_data.shape)\n",
    "print(valid_data.shape)\n",
    "print(test_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "id": "J-xXkD_4175U",
    "outputId": "abd86198-fa3a-4c35-eb10-993337f48b8a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'fear': 0, 'anger': 1, 'sadness': 2, 'joy': 3}\n",
      "['fear', 'anger', 'sadness', 'joy']\n"
     ]
    }
   ],
   "source": [
    "emotions = train_data.emotion.unique()\n",
    "dic = dict()\n",
    "labels = []\n",
    "for i, emotion in enumerate(emotions):\n",
    "    dic[emotion]=i\n",
    "    labels.append(emotion)\n",
    "print(dic)\n",
    "print(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "yhA_GXLyw1gx",
    "outputId": "6a4c4c00-a630-4af6-a75c-fea4d0a33070"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 8672 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "NUM_WORDS=20000\n",
    "texts = train_data.text\n",
    "tokenizer = Tokenizer(num_words=NUM_WORDS,filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n\\'',\n",
    "                      lower=True)\n",
    "tokenizer.fit_on_texts(texts)\n",
    "sequences_train = tokenizer.texts_to_sequences(texts)\n",
    "sequences_valid = tokenizer.texts_to_sequences(valid_data.text)\n",
    "sequences_test = tokenizer.texts_to_sequences(test_data.text)\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "id": "f3a8LX2z0Vn3",
    "outputId": "9fe28fe7-328a-4fcb-9872-437ec9683444"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X train, validation and test tensor: (2610, 33) (291, 33) (2508, 33)\n",
      "Shape of label train, validation and test tensor: (2610, 4) (291, 4) (2508, 4)\n"
     ]
    }
   ],
   "source": [
    "X_train = pad_sequences(sequences_train)\n",
    "X_val = pad_sequences(sequences_valid, maxlen=X_train.shape[1])\n",
    "X_test = pad_sequences(sequences_test, maxlen=X_train.shape[1])\n",
    "\n",
    "y_train = to_categorical(np.asarray(train_data.emotion.apply(lambda x:dic[x])))\n",
    "y_val = to_categorical(np.asarray(valid_data.emotion.apply(lambda x:dic[x])))\n",
    "y_test = to_categorical(np.asarray(test_data.emotion.apply(lambda x:dic[x])))\n",
    "\n",
    "print('Shape of X train, validation and test tensor:', X_train.shape, X_val.shape, X_test.shape)\n",
    "print('Shape of label train, validation and test tensor:', y_train.shape, y_val.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OZprqF6Yqeoe"
   },
   "outputs": [],
   "source": [
    "drive_directory = \"Colab Notebooks\"\n",
    "save_n_upload_numpy_array(X_train, drive_directory, \"emoint_X_train.npy\")\n",
    "save_n_upload_numpy_array(X_val, drive_directory, \"emoint_X_val.npy\")\n",
    "save_n_upload_numpy_array(X_test, drive_directory, \"emoint_X_test.npy\")\n",
    "\n",
    "save_n_upload_numpy_array(y_train, drive_directory, \"emoint_y_train.npy\")\n",
    "save_n_upload_numpy_array(y_val, drive_directory, \"emoint_y_val.npy\")\n",
    "save_n_upload_numpy_array(y_test, drive_directory, \"emoint_y_test.npy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kdMDYkGMF7CA"
   },
   "source": [
    "## Word Embedding using Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WPzDX7qc6eQx"
   },
   "outputs": [],
   "source": [
    "download_drive_file(\"Datasets\", \"GoogleNews-vectors-negative300.bin.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "naD_jPLlmlFq"
   },
   "outputs": [],
   "source": [
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.utils import simple_preprocess\n",
    "\n",
    "from gensim.models.keyedvectors import KeyedVectors\n",
    "\n",
    "word_vectors = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin.gz', binary=True)\n",
    "\n",
    "EMBEDDING_DIM=300\n",
    "vocabulary_size = min(len(word_index)+1, NUM_WORDS)\n",
    "embedding_matrix = np.zeros((vocabulary_size, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    if i>=NUM_WORDS:\n",
    "        continue\n",
    "    try:\n",
    "        embedding_vector = word_vectors[word]\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "    except KeyError:\n",
    "        embedding_matrix[i] = np.random.normal(0,np.sqrt(0.25), EMBEDDING_DIM)\n",
    "\n",
    "del(word_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aXmsV7sr7Ovh"
   },
   "outputs": [],
   "source": [
    "file_embedded_matrix = \"emoint_embedded_matrix_word2vec.npy\"\n",
    "save_n_upload_numpy_array(embedding_matrix, \"Colab Notebooks\", file_embedded_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TD731zTjqTI3"
   },
   "source": [
    "## Reload the embedded layer and training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "SgGl84NR9Y2Y",
    "outputId": "61b12e65-4402-4a12-c7f8-984d7ba226fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "downloading file emoint_embedded_matrix_word2vec.npy, file id 11FmOJypfaaOUqxjfEpdaZW0G7-Qs5yDA\n",
      "downloading file emoint_embedded_matrix_word2vec.npy, file id 1cFCAeIGRXtE_wkW-t_-Z_-ptzSRkavwk\n",
      "embedding matrix shape: (8673, 300)\n"
     ]
    }
   ],
   "source": [
    "file_embedded_matrix = \"emoint_embedded_matrix_word2vec.npy\"\n",
    "download_drive_file(\"Colab Notebooks\", file_embedded_matrix)\n",
    "embedding_matrix = np.load(file_embedded_matrix)\n",
    "print(\"embedding matrix shape:\", embedding_matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 557
    },
    "colab_type": "code",
    "id": "txZDRozUyXAQ",
    "outputId": "ed176b31-9088-4236-d632-83748ce4e04a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "downloading file emoint_X_train.npy, file id 1DNGctnEoSxuOjk4NX95H2sPjO5QJG3zK\n",
      "downloading file emoint_X_train.npy, file id 1Sfzrjds9bwbpidhUj5FU09FX9tPmYSbP\n",
      "downloading file emoint_X_train.npy, file id 1hNAa5pltA2F-lngDrRGsseJymlmJV8uB\n",
      "downloading file emoint_X_train.npy, file id 1IySKG3HZZaBabm8X5_7-HeLhG2h8DwA6\n",
      "downloading file emoint_X_train.npy, file id 1WBHSYesBsMf9-atIMjva6PeTckoQvwNt\n",
      "downloading file emoint_X_val.npy, file id 1gfxnxnUxbFNLgNpXNYGzeR3vcMiAaKlC\n",
      "downloading file emoint_X_val.npy, file id 1wt4-z-kKErILYm-U0jQsFpY9DGQOXryZ\n",
      "downloading file emoint_X_val.npy, file id 1x6EwT9GRrmuwaBrZuSyv0ti59_m16iwB\n",
      "downloading file emoint_X_val.npy, file id 1WL2OKn40zjnZHJA3M0l1caM-udzp0ZOQ\n",
      "downloading file emoint_X_val.npy, file id 1KbkxyL4OLve8_6jpCT3qlLVKr_NcTyxM\n",
      "downloading file emoint_X_test.npy, file id 1XcoZgHVZAxVM_4S-rEcAO15hqhn6AiFr\n",
      "downloading file emoint_X_test.npy, file id 1RfpNialkKT0dBZyvA7v-eRo4WMgZhhql\n",
      "downloading file emoint_X_test.npy, file id 1ulOYckifAMd5pPTNvUyMl0yerS-t_ykj\n",
      "downloading file emoint_X_test.npy, file id 1_Nwt2qXsAoom7tDf0vWwy_OF70O-7pFl\n",
      "downloading file emoint_X_test.npy, file id 10eE9HbzTCnfFsK9LS_Q-PZVXmqZYiIla\n",
      "downloading file emoint_y_train.npy, file id 1WMDn5QcwjeLHQ6m1_ZVhUBkU_ChoLOPD\n",
      "downloading file emoint_y_train.npy, file id 1kk-I_pUH5CraLAO6EiJXUrh7j1bNYANa\n",
      "downloading file emoint_y_train.npy, file id 12O9AYk6-S5V_Q3duVZc_VWXFhY2E5I9x\n",
      "downloading file emoint_y_train.npy, file id 1KMif3hOXmU9mgPvzfowOM1DWpROyGA1J\n",
      "downloading file emoint_y_train.npy, file id 1OaeMr6cgefjv27JIX4oBoCx7Lcc170Pu\n",
      "downloading file emoint_y_val.npy, file id 1jb2jwk4k4FHEyijY1cxZSBlJ0qv7kPVz\n",
      "downloading file emoint_y_val.npy, file id 1sVOajPUodsQDtCOflVlk07ai8msD7FBv\n",
      "downloading file emoint_y_val.npy, file id 14ORBPmqOPbvUmoqNLpHTPwdrIa7PgLZg\n",
      "downloading file emoint_y_val.npy, file id 1gtcwK1eElyn0rSbVODwIygV3CIfrca1i\n",
      "downloading file emoint_y_val.npy, file id 1MdDSJeqKcOamDnQkOV9aozqpHt4zd9Q4\n",
      "downloading file emoint_y_test.npy, file id 1JbcU_ZDHqQmeK7xVTmAz1VaJnGTQri5r\n",
      "downloading file emoint_y_test.npy, file id 1AfPGYYM5V4N_MFgh7-y-XzpcozU6o2f5\n",
      "downloading file emoint_y_test.npy, file id 1VfsjvgbmY3p_T4A2egkchu6kdqmb0Ym2\n",
      "downloading file emoint_y_test.npy, file id 1PiP0Ualc-HmurzFWvvo2Wpps0Jf4-Pt2\n",
      "downloading file emoint_y_test.npy, file id 1M46fsOPEAp1cAwDns2PPK_NESlyN3KUg\n"
     ]
    }
   ],
   "source": [
    "download_drive_file(\"Colab Notebooks\", \"emoint_X_train.npy\")\n",
    "download_drive_file(\"Colab Notebooks\", \"emoint_X_val.npy\")\n",
    "download_drive_file(\"Colab Notebooks\", \"emoint_X_test.npy\")\n",
    "\n",
    "download_drive_file(\"Colab Notebooks\", \"emoint_y_train.npy\")\n",
    "download_drive_file(\"Colab Notebooks\", \"emoint_y_val.npy\")\n",
    "download_drive_file(\"Colab Notebooks\", \"emoint_y_test.npy\")\n",
    "\n",
    "X_train = np.load(\"emoint_X_train.npy\")\n",
    "X_val = np.load(\"emoint_X_val.npy\")\n",
    "X_test = np.load(\"emoint_X_test.npy\")\n",
    "\n",
    "y_train = np.load(\"emoint_y_train.npy\")\n",
    "y_val = np.load(\"emoint_y_val.npy\")\n",
    "y_test = np.load(\"emoint_y_test.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 341
    },
    "colab_type": "code",
    "id": "Q0fX0z7_7Mi5",
    "outputId": "82ec7754-bcbf-44d7-bf24-6a2cd3639bc9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sequence length: 33\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 33)                0         \n",
      "_________________________________________________________________\n",
      "embedding (Embedding)        (None, 33, 300)           2601900   \n",
      "_________________________________________________________________\n",
      "spatial_dropout (SpatialDrop (None, 33, 300)           0         \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 1200)              4324800   \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 4)                 4804      \n",
      "=================================================================\n",
      "Total params: 6,931,504\n",
      "Trainable params: 6,931,504\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dense, Input, GlobalMaxPooling1D\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense, Embedding, Conv2D, MaxPooling2D, Dropout, concatenate, LSTM, SpatialDropout1D, Bidirectional\n",
    "from keras.layers.core import Reshape, Flatten\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.optimizers import Adam, Adadelta, RMSprop\n",
    "from keras.models import Model\n",
    "from keras import regularizers\n",
    "sequence_length = X_train.shape[1]\n",
    "HIDDEN_DIM = 600\n",
    "DROPOUT_RATE = 0.2\n",
    "N_CLASSES = 4\n",
    "vocabulary_size = 8673\n",
    "EMBEDDING_DIM=300\n",
    "\n",
    "print(\"sequence length:\", sequence_length)\n",
    "\n",
    "inputs = Input(shape=(sequence_length,), name='input_1')\n",
    "embedding_layer = Embedding(vocabulary_size, EMBEDDING_DIM, weights=[embedding_matrix], input_length=sequence_length, trainable=True, name='embedding')\n",
    "embedding = embedding_layer(inputs)\n",
    "spatial_droput = SpatialDropout1D(0.4, name='spatial_dropout')(embedding)\n",
    "lstm = Bidirectional(LSTM(units=HIDDEN_DIM, dropout=DROPOUT_RATE, recurrent_dropout=DROPOUT_RATE, name='lstm'))(spatial_droput)\n",
    "output = Dense(units=N_CLASSES, activation='softmax', name='output')(lstm)\n",
    "\n",
    "model = None \n",
    "\n",
    "# this creates a model that includes\n",
    "model = Model(inputs, output)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 395
    },
    "colab_type": "code",
    "id": "rClQ6sed4JA5",
    "outputId": "20296d37-73a1-466f-99fa-fc6504e99985"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 1836376\n",
      "-rw-r--r-- 1 root root       2530 Dec 17 03:53 adc.json\n",
      "drwxr-xr-x 1 root root       4096 Dec 17 03:53 .config\n",
      "-rw-r--r-- 1 root root   55482024 Dec 17 07:15 emoint_bidirectional_lstm_model.h5\n",
      "-rw-r--r-- 1 root root   27745664 Dec 17 07:15 emoint_bidirectional_lstm_weights.h5\n",
      "-rw-r--r-- 1 root root   23748760 Dec 17 07:34 emoint_convnet_model.h5\n",
      "-rw-r--r-- 1 root root   11880960 Dec 17 07:34 emoint_convnet_weights.h5\n",
      "-rw-r--r-- 1 root root   20815328 Dec 17 07:36 emoint_embedded_matrix_word2vec.npy\n",
      "-rw-r--r-- 1 root root   38158376 Dec 17 07:00 emoint_lstm_model.h5\n",
      "-rw-r--r-- 1 root root   19083168 Dec 17 07:00 emoint_lstm_weights.h5\n",
      "-rw-r--r-- 1 root root   23748760 Dec 17 03:57 emoint_model.h5\n",
      "-rw-r--r-- 1 root root   11880960 Dec 17 03:57 emoint_weights.h5\n",
      "-rw-r--r-- 1 root root     331184 Dec 17 07:36 emoint_X_test.npy\n",
      "-rw-r--r-- 1 root root     344648 Dec 17 07:36 emoint_X_train.npy\n",
      "-rw-r--r-- 1 root root      38540 Dec 17 07:36 emoint_X_val.npy\n",
      "-rw-r--r-- 1 root root      40256 Dec 17 07:36 emoint_y_test.npy\n",
      "-rw-r--r-- 1 root root      41888 Dec 17 07:36 emoint_y_train.npy\n",
      "-rw-r--r-- 1 root root       4784 Dec 17 07:36 emoint_y_val.npy\n",
      "-rw-r--r-- 1 root root 1647046227 Dec 17 06:56 GoogleNews-vectors-negative300.bin.gz\n",
      "drwxr-xr-x 1 root root       4096 Dec 10 17:34 sample_data\n",
      "drwxr-xr-x 2 root root       4096 Dec 17 07:37 tmp\n"
     ]
    }
   ],
   "source": [
    "!rm -r tmp\n",
    "!mkdir tmp\n",
    "!ls -lA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oDBXlEv4_0bG"
   },
   "source": [
    "## Two-steps training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 4283
    },
    "colab_type": "code",
    "id": "HHZ1POD77GRp",
    "outputId": "0864429c-6681-43a6-bd68-9bc6099c55e2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2610 samples, validate on 291 samples\n",
      "Epoch 1/100\n",
      "2610/2610 [==============================] - 6s 2ms/step - loss: 1.3674 - acc: 0.3011 - val_loss: 1.2607 - val_acc: 0.5464\n",
      "\n",
      "Epoch 00001: val_acc improved from -inf to 0.54639, saving model to tmp/weights-01-0.55.hdf5\n",
      "Epoch 2/100\n",
      "2610/2610 [==============================] - 2s 905us/step - loss: 1.4675 - acc: 0.4157 - val_loss: 1.3781 - val_acc: 0.2852\n",
      "\n",
      "Epoch 00002: val_acc did not improve from 0.54639\n",
      "Epoch 3/100\n",
      "2610/2610 [==============================] - 2s 911us/step - loss: 1.2841 - acc: 0.4510 - val_loss: 1.2207 - val_acc: 0.5670\n",
      "\n",
      "Epoch 00003: val_acc improved from 0.54639 to 0.56701, saving model to tmp/weights-03-0.57.hdf5\n",
      "Epoch 4/100\n",
      "2610/2610 [==============================] - 2s 900us/step - loss: 1.1777 - acc: 0.5747 - val_loss: 1.1556 - val_acc: 0.5670\n",
      "\n",
      "Epoch 00004: val_acc did not improve from 0.56701\n",
      "Epoch 5/100\n",
      "2610/2610 [==============================] - 2s 909us/step - loss: 1.0818 - acc: 0.5912 - val_loss: 1.1028 - val_acc: 0.5361\n",
      "\n",
      "Epoch 00005: val_acc did not improve from 0.56701\n",
      "Epoch 6/100\n",
      "2610/2610 [==============================] - 2s 907us/step - loss: 1.0680 - acc: 0.5590 - val_loss: 1.0378 - val_acc: 0.6392\n",
      "\n",
      "Epoch 00006: val_acc improved from 0.56701 to 0.63918, saving model to tmp/weights-06-0.64.hdf5\n",
      "Epoch 7/100\n",
      "2610/2610 [==============================] - 2s 906us/step - loss: 0.9340 - acc: 0.6709 - val_loss: 1.1005 - val_acc: 0.5533\n",
      "\n",
      "Epoch 00007: val_acc did not improve from 0.63918\n",
      "Epoch 8/100\n",
      "2610/2610 [==============================] - 2s 919us/step - loss: 0.9525 - acc: 0.6437 - val_loss: 0.8989 - val_acc: 0.7320\n",
      "\n",
      "Epoch 00008: val_acc improved from 0.63918 to 0.73196, saving model to tmp/weights-08-0.73.hdf5\n",
      "Epoch 9/100\n",
      "2610/2610 [==============================] - 2s 910us/step - loss: 0.8017 - acc: 0.7287 - val_loss: 0.8872 - val_acc: 0.6942\n",
      "\n",
      "Epoch 00009: val_acc did not improve from 0.73196\n",
      "Epoch 10/100\n",
      "2610/2610 [==============================] - 2s 913us/step - loss: 0.7522 - acc: 0.7203 - val_loss: 0.7782 - val_acc: 0.7423\n",
      "\n",
      "Epoch 00010: val_acc improved from 0.73196 to 0.74227, saving model to tmp/weights-10-0.74.hdf5\n",
      "Epoch 11/100\n",
      "2610/2610 [==============================] - 2s 919us/step - loss: 0.6229 - acc: 0.7766 - val_loss: 0.8110 - val_acc: 0.7010\n",
      "\n",
      "Epoch 00011: val_acc did not improve from 0.74227\n",
      "Epoch 12/100\n",
      "2610/2610 [==============================] - 2s 916us/step - loss: 0.6397 - acc: 0.7774 - val_loss: 0.6983 - val_acc: 0.7491\n",
      "\n",
      "Epoch 00012: val_acc improved from 0.74227 to 0.74914, saving model to tmp/weights-12-0.75.hdf5\n",
      "Epoch 13/100\n",
      "2610/2610 [==============================] - 2s 908us/step - loss: 0.5654 - acc: 0.8034 - val_loss: 0.6730 - val_acc: 0.8110\n",
      "\n",
      "Epoch 00013: val_acc improved from 0.74914 to 0.81100, saving model to tmp/weights-13-0.81.hdf5\n",
      "Epoch 14/100\n",
      "2610/2610 [==============================] - 2s 912us/step - loss: 1.2294 - acc: 0.7238 - val_loss: 1.8567 - val_acc: 0.5258\n",
      "\n",
      "Epoch 00014: val_acc did not improve from 0.81100\n",
      "Epoch 15/100\n",
      "2610/2610 [==============================] - 2s 925us/step - loss: 1.2054 - acc: 0.6467 - val_loss: 0.8155 - val_acc: 0.7216\n",
      "\n",
      "Epoch 00015: val_acc did not improve from 0.81100\n",
      "Epoch 16/100\n",
      "2610/2610 [==============================] - 2s 919us/step - loss: 0.6975 - acc: 0.7594 - val_loss: 0.7286 - val_acc: 0.7732\n",
      "\n",
      "Epoch 00016: val_acc did not improve from 0.81100\n",
      "Epoch 17/100\n",
      "2610/2610 [==============================] - 2s 921us/step - loss: 0.5539 - acc: 0.8253 - val_loss: 0.6374 - val_acc: 0.8144\n",
      "\n",
      "Epoch 00017: val_acc improved from 0.81100 to 0.81443, saving model to tmp/weights-17-0.81.hdf5\n",
      "Epoch 18/100\n",
      "2610/2610 [==============================] - 2s 915us/step - loss: 0.4618 - acc: 0.8460 - val_loss: 0.6043 - val_acc: 0.8110\n",
      "\n",
      "Epoch 00018: val_acc did not improve from 0.81443\n",
      "Epoch 19/100\n",
      "2610/2610 [==============================] - 2s 913us/step - loss: 0.3732 - acc: 0.8835 - val_loss: 0.5922 - val_acc: 0.7973\n",
      "\n",
      "Epoch 00019: val_acc did not improve from 0.81443\n",
      "Epoch 20/100\n",
      "2610/2610 [==============================] - 2s 908us/step - loss: 0.4054 - acc: 0.8709 - val_loss: 0.5663 - val_acc: 0.8351\n",
      "\n",
      "Epoch 00020: val_acc improved from 0.81443 to 0.83505, saving model to tmp/weights-20-0.84.hdf5\n",
      "Epoch 21/100\n",
      "2610/2610 [==============================] - 2s 911us/step - loss: 0.3161 - acc: 0.9011 - val_loss: 0.5032 - val_acc: 0.8488\n",
      "\n",
      "Epoch 00021: val_acc improved from 0.83505 to 0.84880, saving model to tmp/weights-21-0.85.hdf5\n",
      "Epoch 22/100\n",
      "2610/2610 [==============================] - 2s 919us/step - loss: 0.2699 - acc: 0.9153 - val_loss: 0.5149 - val_acc: 0.8351\n",
      "\n",
      "Epoch 00022: val_acc did not improve from 0.84880\n",
      "Epoch 23/100\n",
      "2610/2610 [==============================] - 2s 919us/step - loss: 0.3212 - acc: 0.8839 - val_loss: 0.5486 - val_acc: 0.7938\n",
      "\n",
      "Epoch 00023: val_acc did not improve from 0.84880\n",
      "Epoch 24/100\n",
      "2610/2610 [==============================] - 2s 918us/step - loss: 0.3179 - acc: 0.8969 - val_loss: 0.4939 - val_acc: 0.8522\n",
      "\n",
      "Epoch 00024: val_acc improved from 0.84880 to 0.85223, saving model to tmp/weights-24-0.85.hdf5\n",
      "Epoch 25/100\n",
      "2610/2610 [==============================] - 2s 912us/step - loss: 0.2199 - acc: 0.9429 - val_loss: 0.4999 - val_acc: 0.8625\n",
      "\n",
      "Epoch 00025: val_acc improved from 0.85223 to 0.86254, saving model to tmp/weights-25-0.86.hdf5\n",
      "Epoch 26/100\n",
      "2610/2610 [==============================] - 2s 917us/step - loss: 0.2000 - acc: 0.9349 - val_loss: 0.5550 - val_acc: 0.8110\n",
      "\n",
      "Epoch 00026: val_acc did not improve from 0.86254\n",
      "Epoch 27/100\n",
      "2610/2610 [==============================] - 2s 922us/step - loss: 0.2423 - acc: 0.9261 - val_loss: 0.4439 - val_acc: 0.8660\n",
      "\n",
      "Epoch 00027: val_acc improved from 0.86254 to 0.86598, saving model to tmp/weights-27-0.87.hdf5\n",
      "Epoch 28/100\n",
      "2610/2610 [==============================] - 2s 912us/step - loss: 0.2091 - acc: 0.9299 - val_loss: 0.5190 - val_acc: 0.8316\n",
      "\n",
      "Epoch 00028: val_acc did not improve from 0.86598\n",
      "Epoch 29/100\n",
      "2610/2610 [==============================] - 2s 919us/step - loss: 0.2228 - acc: 0.9238 - val_loss: 0.4990 - val_acc: 0.8385\n",
      "\n",
      "Epoch 00029: val_acc did not improve from 0.86598\n",
      "Epoch 30/100\n",
      "2610/2610 [==============================] - 2s 914us/step - loss: 0.1744 - acc: 0.9456 - val_loss: 0.4302 - val_acc: 0.8694\n",
      "\n",
      "Epoch 00030: val_acc improved from 0.86598 to 0.86942, saving model to tmp/weights-30-0.87.hdf5\n",
      "Epoch 31/100\n",
      "2610/2610 [==============================] - 2s 910us/step - loss: 0.1633 - acc: 0.9475 - val_loss: 0.4921 - val_acc: 0.8660\n",
      "\n",
      "Epoch 00031: val_acc did not improve from 0.86942\n",
      "Epoch 32/100\n",
      "2610/2610 [==============================] - 2s 913us/step - loss: 0.2084 - acc: 0.9299 - val_loss: 0.5224 - val_acc: 0.8282\n",
      "\n",
      "Epoch 00032: val_acc did not improve from 0.86942\n",
      "Epoch 33/100\n",
      "2610/2610 [==============================] - 2s 915us/step - loss: 0.1997 - acc: 0.9349 - val_loss: 0.4527 - val_acc: 0.8591\n",
      "\n",
      "Epoch 00033: val_acc did not improve from 0.86942\n",
      "Epoch 34/100\n",
      "2610/2610 [==============================] - 2s 912us/step - loss: 0.1244 - acc: 0.9617 - val_loss: 0.4742 - val_acc: 0.8660\n",
      "\n",
      "Epoch 00034: val_acc did not improve from 0.86942\n",
      "Epoch 35/100\n",
      "2610/2610 [==============================] - 2s 916us/step - loss: 0.1635 - acc: 0.9471 - val_loss: 0.4465 - val_acc: 0.8557\n",
      "\n",
      "Epoch 00035: val_acc did not improve from 0.86942\n",
      "Epoch 36/100\n",
      "2610/2610 [==============================] - 2s 916us/step - loss: 0.1251 - acc: 0.9655 - val_loss: 0.4355 - val_acc: 0.8763\n",
      "\n",
      "Epoch 00036: val_acc improved from 0.86942 to 0.87629, saving model to tmp/weights-36-0.88.hdf5\n",
      "Epoch 37/100\n",
      "2610/2610 [==============================] - 2s 910us/step - loss: 0.0917 - acc: 0.9716 - val_loss: 0.4438 - val_acc: 0.8522\n",
      "\n",
      "Epoch 00037: val_acc did not improve from 0.87629\n",
      "Epoch 38/100\n",
      "2610/2610 [==============================] - 2s 913us/step - loss: 0.1381 - acc: 0.9490 - val_loss: 0.4311 - val_acc: 0.8694\n",
      "\n",
      "Epoch 00038: val_acc did not improve from 0.87629\n",
      "Epoch 39/100\n",
      "2610/2610 [==============================] - 2s 915us/step - loss: 0.1281 - acc: 0.9594 - val_loss: 0.4184 - val_acc: 0.8729\n",
      "\n",
      "Epoch 00039: val_acc did not improve from 0.87629\n",
      "Epoch 40/100\n",
      "2610/2610 [==============================] - 2s 922us/step - loss: 0.0861 - acc: 0.9713 - val_loss: 0.4756 - val_acc: 0.8660\n",
      "\n",
      "Epoch 00040: val_acc did not improve from 0.87629\n",
      "Epoch 41/100\n",
      "2610/2610 [==============================] - 2s 916us/step - loss: 0.0991 - acc: 0.9716 - val_loss: 0.4202 - val_acc: 0.8866\n",
      "\n",
      "Epoch 00041: val_acc improved from 0.87629 to 0.88660, saving model to tmp/weights-41-0.89.hdf5\n",
      "Epoch 42/100\n",
      "2610/2610 [==============================] - 2s 909us/step - loss: 0.0697 - acc: 0.9793 - val_loss: 0.4331 - val_acc: 0.8591\n",
      "\n",
      "Epoch 00042: val_acc did not improve from 0.88660\n",
      "Epoch 43/100\n",
      "2610/2610 [==============================] - 2s 915us/step - loss: 0.0849 - acc: 0.9705 - val_loss: 0.5424 - val_acc: 0.8385\n",
      "\n",
      "Epoch 00043: val_acc did not improve from 0.88660\n",
      "Epoch 44/100\n",
      "2610/2610 [==============================] - 2s 917us/step - loss: 0.1479 - acc: 0.9506 - val_loss: 0.4751 - val_acc: 0.8351\n",
      "\n",
      "Epoch 00044: val_acc did not improve from 0.88660\n",
      "Epoch 45/100\n",
      "2610/2610 [==============================] - 2s 916us/step - loss: 0.1113 - acc: 0.9667 - val_loss: 0.4260 - val_acc: 0.8797\n",
      "\n",
      "Epoch 00045: val_acc did not improve from 0.88660\n",
      "Epoch 46/100\n",
      "2610/2610 [==============================] - 2s 917us/step - loss: 0.0639 - acc: 0.9778 - val_loss: 0.4316 - val_acc: 0.8935\n",
      "\n",
      "Epoch 00046: val_acc improved from 0.88660 to 0.89347, saving model to tmp/weights-46-0.89.hdf5\n",
      "Epoch 47/100\n",
      "2610/2610 [==============================] - 2s 915us/step - loss: 0.0756 - acc: 0.9716 - val_loss: 0.4358 - val_acc: 0.8625\n",
      "\n",
      "Epoch 00047: val_acc did not improve from 0.89347\n",
      "Epoch 48/100\n",
      "2610/2610 [==============================] - 2s 916us/step - loss: 0.0818 - acc: 0.9736 - val_loss: 0.4223 - val_acc: 0.8797\n",
      "\n",
      "Epoch 00048: val_acc did not improve from 0.89347\n",
      "Epoch 49/100\n",
      "2610/2610 [==============================] - 2s 921us/step - loss: 0.0577 - acc: 0.9805 - val_loss: 0.4052 - val_acc: 0.8832\n",
      "\n",
      "Epoch 00049: val_acc did not improve from 0.89347\n",
      "Epoch 50/100\n",
      "2610/2610 [==============================] - 2s 914us/step - loss: 0.0719 - acc: 0.9770 - val_loss: 0.4686 - val_acc: 0.8660\n",
      "\n",
      "Epoch 00050: val_acc did not improve from 0.89347\n",
      "Epoch 51/100\n",
      "2610/2610 [==============================] - 2s 916us/step - loss: 0.0665 - acc: 0.9793 - val_loss: 0.4233 - val_acc: 0.8660\n",
      "\n",
      "Epoch 00051: val_acc did not improve from 0.89347\n",
      "Epoch 52/100\n",
      "2610/2610 [==============================] - 2s 915us/step - loss: 0.0541 - acc: 0.9839 - val_loss: 0.4655 - val_acc: 0.8694\n",
      "\n",
      "Epoch 00052: val_acc did not improve from 0.89347\n",
      "Epoch 53/100\n",
      "2610/2610 [==============================] - 2s 915us/step - loss: 0.0495 - acc: 0.9847 - val_loss: 0.4663 - val_acc: 0.8625\n",
      "\n",
      "Epoch 00053: val_acc did not improve from 0.89347\n",
      "Epoch 54/100\n",
      "2610/2610 [==============================] - 2s 912us/step - loss: 0.0440 - acc: 0.9866 - val_loss: 0.4538 - val_acc: 0.8763\n",
      "\n",
      "Epoch 00054: val_acc did not improve from 0.89347\n",
      "Epoch 55/100\n",
      "2610/2610 [==============================] - 2s 917us/step - loss: 0.0478 - acc: 0.9851 - val_loss: 0.4293 - val_acc: 0.8694\n",
      "\n",
      "Epoch 00055: val_acc did not improve from 0.89347\n",
      "Epoch 56/100\n",
      "2610/2610 [==============================] - 2s 910us/step - loss: 0.0734 - acc: 0.9766 - val_loss: 0.4473 - val_acc: 0.8660\n",
      "\n",
      "Epoch 00056: val_acc did not improve from 0.89347\n",
      "Epoch 57/100\n",
      "2610/2610 [==============================] - 2s 913us/step - loss: 0.0496 - acc: 0.9820 - val_loss: 0.4502 - val_acc: 0.8694\n",
      "\n",
      "Epoch 00057: val_acc did not improve from 0.89347\n",
      "Epoch 58/100\n",
      "2610/2610 [==============================] - 2s 915us/step - loss: 0.0939 - acc: 0.9651 - val_loss: 0.4486 - val_acc: 0.8729\n",
      "\n",
      "Epoch 00058: val_acc did not improve from 0.89347\n",
      "Epoch 59/100\n",
      "2610/2610 [==============================] - 2s 920us/step - loss: 0.0494 - acc: 0.9847 - val_loss: 0.4080 - val_acc: 0.8866\n",
      "\n",
      "Epoch 00059: val_acc did not improve from 0.89347\n"
     ]
    }
   ],
   "source": [
    "adam = Adam(lr=1e-3, decay=0.0)\n",
    "adadelta = Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)\n",
    "rmsprop = RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer=rmsprop, metrics=['acc'])\n",
    "\n",
    "filepath=\"tmp/weights-{epoch:02d}-{val_acc:.2f}.hdf5\"\n",
    "model_checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='auto')\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=10)\n",
    "callbacks = [early_stopping, model_checkpoint]\n",
    "\n",
    "history = model.fit(X_train, y_train, batch_size=1000, epochs=100, verbose=1, validation_data=(X_val, y_val), callbacks=callbacks)  # starts training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 3131
    },
    "colab_type": "code",
    "id": "8yc2upLU_4Zl",
    "outputId": "a7ddec7e-dee1-4d2f-a9ca-777efc145578"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2901 samples, validate on 291 samples\n",
      "Epoch 1/100\n",
      "2901/2901 [==============================] - 3s 896us/step - loss: 0.1000 - acc: 0.9686 - val_loss: 0.4029 - val_acc: 0.8866\n",
      "\n",
      "Epoch 00001: val_acc did not improve from 0.89347\n",
      "Epoch 2/100\n",
      "2901/2901 [==============================] - 3s 894us/step - loss: 0.1124 - acc: 0.9593 - val_loss: 0.2709 - val_acc: 0.9210\n",
      "\n",
      "Epoch 00002: val_acc improved from 0.89347 to 0.92096, saving model to tmp/weights-02-0.92.hdf5\n",
      "Epoch 3/100\n",
      "2901/2901 [==============================] - 3s 894us/step - loss: 0.0794 - acc: 0.9717 - val_loss: 0.2203 - val_acc: 0.9313\n",
      "\n",
      "Epoch 00003: val_acc improved from 0.92096 to 0.93127, saving model to tmp/weights-03-0.93.hdf5\n",
      "Epoch 4/100\n",
      "2901/2901 [==============================] - 3s 898us/step - loss: 0.0658 - acc: 0.9741 - val_loss: 0.1913 - val_acc: 0.9416\n",
      "\n",
      "Epoch 00004: val_acc improved from 0.93127 to 0.94158, saving model to tmp/weights-04-0.94.hdf5\n",
      "Epoch 5/100\n",
      "2901/2901 [==============================] - 3s 892us/step - loss: 0.0605 - acc: 0.9797 - val_loss: 0.1588 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00005: val_acc improved from 0.94158 to 0.94502, saving model to tmp/weights-05-0.95.hdf5\n",
      "Epoch 6/100\n",
      "2901/2901 [==============================] - 3s 893us/step - loss: 0.0699 - acc: 0.9738 - val_loss: 0.1794 - val_acc: 0.9485\n",
      "\n",
      "Epoch 00006: val_acc improved from 0.94502 to 0.94845, saving model to tmp/weights-06-0.95.hdf5\n",
      "Epoch 7/100\n",
      "2901/2901 [==============================] - 3s 894us/step - loss: 0.0569 - acc: 0.9835 - val_loss: 0.1201 - val_acc: 0.9553\n",
      "\n",
      "Epoch 00007: val_acc improved from 0.94845 to 0.95533, saving model to tmp/weights-07-0.96.hdf5\n",
      "Epoch 8/100\n",
      "2901/2901 [==============================] - 3s 893us/step - loss: 0.0560 - acc: 0.9800 - val_loss: 0.1147 - val_acc: 0.9553\n",
      "\n",
      "Epoch 00008: val_acc did not improve from 0.95533\n",
      "Epoch 9/100\n",
      "2901/2901 [==============================] - 3s 896us/step - loss: 0.0541 - acc: 0.9807 - val_loss: 0.1114 - val_acc: 0.9485\n",
      "\n",
      "Epoch 00009: val_acc did not improve from 0.95533\n",
      "Epoch 10/100\n",
      "2901/2901 [==============================] - 3s 898us/step - loss: 0.0588 - acc: 0.9817 - val_loss: 0.0971 - val_acc: 0.9656\n",
      "\n",
      "Epoch 00010: val_acc improved from 0.95533 to 0.96564, saving model to tmp/weights-10-0.97.hdf5\n",
      "Epoch 11/100\n",
      "2901/2901 [==============================] - 3s 897us/step - loss: 0.0488 - acc: 0.9817 - val_loss: 0.0839 - val_acc: 0.9691\n",
      "\n",
      "Epoch 00011: val_acc improved from 0.96564 to 0.96907, saving model to tmp/weights-11-0.97.hdf5\n",
      "Epoch 12/100\n",
      "2901/2901 [==============================] - 3s 893us/step - loss: 0.0671 - acc: 0.9762 - val_loss: 0.0877 - val_acc: 0.9656\n",
      "\n",
      "Epoch 00012: val_acc did not improve from 0.96907\n",
      "Epoch 13/100\n",
      "2901/2901 [==============================] - 3s 893us/step - loss: 0.0443 - acc: 0.9835 - val_loss: 0.0695 - val_acc: 0.9691\n",
      "\n",
      "Epoch 00013: val_acc did not improve from 0.96907\n",
      "Epoch 14/100\n",
      "2901/2901 [==============================] - 3s 898us/step - loss: 0.0423 - acc: 0.9848 - val_loss: 0.0705 - val_acc: 0.9725\n",
      "\n",
      "Epoch 00014: val_acc improved from 0.96907 to 0.97251, saving model to tmp/weights-14-0.97.hdf5\n",
      "Epoch 15/100\n",
      "2901/2901 [==============================] - 3s 895us/step - loss: 0.0479 - acc: 0.9821 - val_loss: 0.0778 - val_acc: 0.9691\n",
      "\n",
      "Epoch 00015: val_acc did not improve from 0.97251\n",
      "Epoch 16/100\n",
      "2901/2901 [==============================] - 3s 898us/step - loss: 0.0436 - acc: 0.9859 - val_loss: 0.0512 - val_acc: 0.9794\n",
      "\n",
      "Epoch 00016: val_acc improved from 0.97251 to 0.97938, saving model to tmp/weights-16-0.98.hdf5\n",
      "Epoch 17/100\n",
      "2901/2901 [==============================] - 3s 895us/step - loss: 0.0512 - acc: 0.9786 - val_loss: 0.0615 - val_acc: 0.9828\n",
      "\n",
      "Epoch 00017: val_acc improved from 0.97938 to 0.98282, saving model to tmp/weights-17-0.98.hdf5\n",
      "Epoch 18/100\n",
      "2901/2901 [==============================] - 3s 897us/step - loss: 0.0711 - acc: 0.9755 - val_loss: 0.0883 - val_acc: 0.9759\n",
      "\n",
      "Epoch 00018: val_acc did not improve from 0.98282\n",
      "Epoch 19/100\n",
      "2901/2901 [==============================] - 3s 904us/step - loss: 0.0430 - acc: 0.9838 - val_loss: 0.0573 - val_acc: 0.9794\n",
      "\n",
      "Epoch 00019: val_acc did not improve from 0.98282\n",
      "Epoch 20/100\n",
      "2901/2901 [==============================] - 3s 895us/step - loss: 0.0364 - acc: 0.9835 - val_loss: 0.0460 - val_acc: 0.9759\n",
      "\n",
      "Epoch 00020: val_acc did not improve from 0.98282\n",
      "Epoch 21/100\n",
      "2901/2901 [==============================] - 3s 898us/step - loss: 0.0401 - acc: 0.9848 - val_loss: 0.0586 - val_acc: 0.9759\n",
      "\n",
      "Epoch 00021: val_acc did not improve from 0.98282\n",
      "Epoch 22/100\n",
      "2901/2901 [==============================] - 3s 897us/step - loss: 0.0399 - acc: 0.9824 - val_loss: 0.0368 - val_acc: 0.9897\n",
      "\n",
      "Epoch 00022: val_acc improved from 0.98282 to 0.98969, saving model to tmp/weights-22-0.99.hdf5\n",
      "Epoch 23/100\n",
      "2901/2901 [==============================] - 3s 895us/step - loss: 0.0377 - acc: 0.9841 - val_loss: 0.0369 - val_acc: 0.9828\n",
      "\n",
      "Epoch 00023: val_acc did not improve from 0.98969\n",
      "Epoch 24/100\n",
      "2901/2901 [==============================] - 3s 890us/step - loss: 0.0341 - acc: 0.9859 - val_loss: 0.0553 - val_acc: 0.9725\n",
      "\n",
      "Epoch 00024: val_acc did not improve from 0.98969\n",
      "Epoch 25/100\n",
      "2901/2901 [==============================] - 3s 895us/step - loss: 0.0485 - acc: 0.9821 - val_loss: 0.0503 - val_acc: 0.9794\n",
      "\n",
      "Epoch 00025: val_acc did not improve from 0.98969\n",
      "Epoch 26/100\n",
      "2901/2901 [==============================] - 3s 896us/step - loss: 0.0361 - acc: 0.9866 - val_loss: 0.0471 - val_acc: 0.9725\n",
      "\n",
      "Epoch 00026: val_acc did not improve from 0.98969\n",
      "Epoch 27/100\n",
      "2901/2901 [==============================] - 3s 897us/step - loss: 0.0287 - acc: 0.9876 - val_loss: 0.0384 - val_acc: 0.9794\n",
      "\n",
      "Epoch 00027: val_acc did not improve from 0.98969\n",
      "Epoch 28/100\n",
      "2901/2901 [==============================] - 3s 895us/step - loss: 0.0349 - acc: 0.9855 - val_loss: 0.0435 - val_acc: 0.9828\n",
      "\n",
      "Epoch 00028: val_acc did not improve from 0.98969\n",
      "Epoch 29/100\n",
      "2901/2901 [==============================] - 3s 898us/step - loss: 0.0432 - acc: 0.9841 - val_loss: 0.0446 - val_acc: 0.9828\n",
      "\n",
      "Epoch 00029: val_acc did not improve from 0.98969\n",
      "Epoch 30/100\n",
      "2901/2901 [==============================] - 3s 897us/step - loss: 0.0347 - acc: 0.9852 - val_loss: 0.0349 - val_acc: 0.9828\n",
      "\n",
      "Epoch 00030: val_acc did not improve from 0.98969\n",
      "Epoch 31/100\n",
      "2901/2901 [==============================] - 3s 895us/step - loss: 0.0351 - acc: 0.9828 - val_loss: 0.0525 - val_acc: 0.9794\n",
      "\n",
      "Epoch 00031: val_acc did not improve from 0.98969\n",
      "Epoch 32/100\n",
      "2901/2901 [==============================] - 3s 894us/step - loss: 0.0439 - acc: 0.9835 - val_loss: 0.0338 - val_acc: 0.9863\n",
      "\n",
      "Epoch 00032: val_acc did not improve from 0.98969\n",
      "Epoch 33/100\n",
      "2901/2901 [==============================] - 3s 894us/step - loss: 0.0362 - acc: 0.9828 - val_loss: 0.0318 - val_acc: 0.9863\n",
      "\n",
      "Epoch 00033: val_acc did not improve from 0.98969\n",
      "Epoch 34/100\n",
      "2901/2901 [==============================] - 3s 898us/step - loss: 0.0311 - acc: 0.9879 - val_loss: 0.0339 - val_acc: 0.9794\n",
      "\n",
      "Epoch 00034: val_acc did not improve from 0.98969\n",
      "Epoch 35/100\n",
      "2901/2901 [==============================] - 3s 900us/step - loss: 0.0327 - acc: 0.9838 - val_loss: 0.0351 - val_acc: 0.9828\n",
      "\n",
      "Epoch 00035: val_acc did not improve from 0.98969\n",
      "Epoch 36/100\n",
      "2901/2901 [==============================] - 3s 892us/step - loss: 0.0284 - acc: 0.9862 - val_loss: 0.0323 - val_acc: 0.9828\n",
      "\n",
      "Epoch 00036: val_acc did not improve from 0.98969\n",
      "Epoch 37/100\n",
      "2901/2901 [==============================] - 3s 900us/step - loss: 0.0293 - acc: 0.9866 - val_loss: 0.0379 - val_acc: 0.9759\n",
      "\n",
      "Epoch 00037: val_acc did not improve from 0.98969\n",
      "Epoch 38/100\n",
      "2901/2901 [==============================] - 3s 897us/step - loss: 0.0319 - acc: 0.9855 - val_loss: 0.0426 - val_acc: 0.9759\n",
      "\n",
      "Epoch 00038: val_acc did not improve from 0.98969\n",
      "Epoch 39/100\n",
      "2901/2901 [==============================] - 3s 898us/step - loss: 0.0396 - acc: 0.9817 - val_loss: 0.0687 - val_acc: 0.9691\n",
      "\n",
      "Epoch 00039: val_acc did not improve from 0.98969\n",
      "Epoch 40/100\n",
      "2901/2901 [==============================] - 3s 898us/step - loss: 0.0581 - acc: 0.9786 - val_loss: 0.0511 - val_acc: 0.9794\n",
      "\n",
      "Epoch 00040: val_acc did not improve from 0.98969\n",
      "Epoch 41/100\n",
      "2901/2901 [==============================] - 3s 900us/step - loss: 0.0401 - acc: 0.9835 - val_loss: 0.0351 - val_acc: 0.9794\n",
      "\n",
      "Epoch 00041: val_acc did not improve from 0.98969\n",
      "Epoch 42/100\n",
      "2901/2901 [==============================] - 3s 893us/step - loss: 0.0310 - acc: 0.9859 - val_loss: 0.0360 - val_acc: 0.9794\n",
      "\n",
      "Epoch 00042: val_acc did not improve from 0.98969\n",
      "Epoch 43/100\n",
      "2901/2901 [==============================] - 3s 893us/step - loss: 0.0306 - acc: 0.9855 - val_loss: 0.0414 - val_acc: 0.9759\n",
      "\n",
      "Epoch 00043: val_acc did not improve from 0.98969\n"
     ]
    }
   ],
   "source": [
    "X_train_combined = np.concatenate((X_train, X_val))\n",
    "y_train_combined = np.concatenate((y_train, y_val))\n",
    "\n",
    "history = model.fit(X_train_combined, y_train_combined, batch_size=1000, epochs=100, verbose=1, validation_data=(X_val, y_val), callbacks=callbacks)  # starts training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OtGrA8RBKShg"
   },
   "outputs": [],
   "source": [
    "model_path = 'emoint_bidirectional_lstm_model.h5'\n",
    "weight_path = 'emoint_bidirectional_lstm_weights.h5'\n",
    "\n",
    "model.save(model_path)\n",
    "model.save_weights(weight_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 431
    },
    "colab_type": "code",
    "id": "eFzd1urtKeFw",
    "outputId": "01a74e98-8890-44ae-caa8-7cd95bcaa8b2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 1.8G\n",
      "drwxr-xr-x 1 root root 4.0K Dec 17 07:37 .\n",
      "drwxr-xr-x 1 root root 4.0K Dec 17 03:52 ..\n",
      "-rw-r--r-- 1 root root 2.5K Dec 17 03:53 adc.json\n",
      "drwxr-xr-x 1 root root 4.0K Dec 17 03:53 .config\n",
      "-rw-r--r-- 1 root root  53M Dec 17 07:41 emoint_bidirectional_lstm_model.h5\n",
      "-rw-r--r-- 1 root root  27M Dec 17 07:41 emoint_bidirectional_lstm_weights.h5\n",
      "-rw-r--r-- 1 root root  23M Dec 17 07:34 emoint_convnet_model.h5\n",
      "-rw-r--r-- 1 root root  12M Dec 17 07:34 emoint_convnet_weights.h5\n",
      "-rw-r--r-- 1 root root  20M Dec 17 07:36 emoint_embedded_matrix_word2vec.npy\n",
      "-rw-r--r-- 1 root root  37M Dec 17 07:00 emoint_lstm_model.h5\n",
      "-rw-r--r-- 1 root root  19M Dec 17 07:00 emoint_lstm_weights.h5\n",
      "-rw-r--r-- 1 root root  23M Dec 17 03:57 emoint_model.h5\n",
      "-rw-r--r-- 1 root root  12M Dec 17 03:57 emoint_weights.h5\n",
      "-rw-r--r-- 1 root root 324K Dec 17 07:36 emoint_X_test.npy\n",
      "-rw-r--r-- 1 root root 337K Dec 17 07:36 emoint_X_train.npy\n",
      "-rw-r--r-- 1 root root  38K Dec 17 07:36 emoint_X_val.npy\n",
      "-rw-r--r-- 1 root root  40K Dec 17 07:36 emoint_y_test.npy\n",
      "-rw-r--r-- 1 root root  41K Dec 17 07:36 emoint_y_train.npy\n",
      "-rw-r--r-- 1 root root 4.7K Dec 17 07:36 emoint_y_val.npy\n",
      "-rw-r--r-- 1 root root 1.6G Dec 17 06:56 GoogleNews-vectors-negative300.bin.gz\n",
      "drwxr-xr-x 1 root root 4.0K Dec 10 17:34 sample_data\n",
      "drwxr-xr-x 2 root root 4.0K Dec 17 07:40 tmp\n"
     ]
    }
   ],
   "source": [
    "!ls -lah"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "85MiexxMK8zd"
   },
   "outputs": [],
   "source": [
    "upload_drive_file(\"Colab Notebooks\", model_path)\n",
    "upload_drive_file(\"Colab Notebooks\", weight_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b0P0vTv-7RC6"
   },
   "outputs": [],
   "source": [
    "sequences_test=tokenizer.texts_to_sequences(test_data.text)\n",
    "X_test = pad_sequences(sequences_test, maxlen=X_test.shape[1])\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "IgHlDfC_-rS5",
    "outputId": "f3945581-42d0-42a1-b0df-4d91d9f7e6e8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['val_loss', 'val_acc', 'loss', 'acc'])"
      ]
     },
     "execution_count": 23,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-JaFoS4u_pwS"
   },
   "outputs": [],
   "source": [
    "y_pred_original = [labels[val] for val in np.argmax(y_pred, axis=1).squeeze()]\n",
    "y_test_original = np.asarray(test_data.emotion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BREyqdCAI9d0"
   },
   "outputs": [],
   "source": [
    "def print_confusion_matrix(confusion_matrix, class_names, figsize = (4,3), fontsize=15):\n",
    "    \"\"\"Prints a confusion matrix, as returned by sklearn.metrics.confusion_matrix, as a heatmap.\n",
    "    \n",
    "    Arguments\n",
    "    ---------\n",
    "    confusion_matrix: numpy.ndarray\n",
    "        The numpy.ndarray object returned from a call to sklearn.metrics.confusion_matrix. \n",
    "        Similarly constructed ndarrays can also be used.\n",
    "    class_names: list\n",
    "        An ordered list of class names, in the order they index the given confusion matrix.\n",
    "    figsize: tuple\n",
    "        A 2-long tuple, the first value determining the horizontal size of the ouputted figure,\n",
    "        the second determining the vertical size. Defaults to (10,7).\n",
    "    fontsize: int\n",
    "        Font size for axes labels. Defaults to 14.\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    matplotlib.figure.Figure\n",
    "        The resulting confusion matrix figure\n",
    "    \"\"\"\n",
    "    df_cm = pd.DataFrame(\n",
    "        confusion_matrix, index=class_names, columns=class_names, \n",
    "    )\n",
    "    fig = plt.figure(figsize=figsize)\n",
    "    try:\n",
    "        heatmap = sns.heatmap(df_cm, annot=True, fmt=\"d\")\n",
    "    except ValueError:\n",
    "        raise ValueError(\"Confusion matrix values must be integers.\")\n",
    "    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=fontsize)\n",
    "    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=fontsize)\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 173
    },
    "colab_type": "code",
    "id": "XG9HPQb10TZm",
    "outputId": "c9fe229c-43fe-41bd-a3d6-82c163c1ad2f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fear</th>\n",
       "      <th>anger</th>\n",
       "      <th>sadness</th>\n",
       "      <th>joy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fear</th>\n",
       "      <td>638</td>\n",
       "      <td>51</td>\n",
       "      <td>78</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>65</td>\n",
       "      <td>520</td>\n",
       "      <td>46</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sadness</th>\n",
       "      <td>69</td>\n",
       "      <td>22</td>\n",
       "      <td>428</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>joy</th>\n",
       "      <td>26</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>492</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         fear  anger  sadness  joy\n",
       "fear      638     51       78   22\n",
       "anger      65    520       46   12\n",
       "sadness    69     22      428   14\n",
       "joy        26     11       14  492"
      ]
     },
     "execution_count": 26,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cf_matrix = confusion_matrix(y_test_original, y_pred_original, labels=labels)\n",
    "\n",
    "df_cm = pd.DataFrame(\n",
    "    cf_matrix, index=labels, columns=labels, \n",
    ")\n",
    "\n",
    "df_cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 282
    },
    "colab_type": "code",
    "id": "59xnXmbEAKXy",
    "outputId": "9f68f9f9-0b05-46df-c050-6be4554b0013"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Figure(288x216)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATkAAAD3CAYAAABmdCOBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XdYFNcexvHvsoB0FQSsiFiwUBS9\nNtBYMHbsxB4Vey8o9hI11mABBRWxBCvYNXbFClijGDsaRWw0Gx127h9eV7lGUBF2Jedzn30uOzs7\n+2LgxzlnzpyRSZIkIQiCkE9pqDqAIAhCbhJFThCEfE0UOUEQ8jVR5ARByNdEkRMEIV/TVHWA/Mau\n9A+qjvBFzp5areoIXyw9IUHVEb6Ijpm5qiN8FR2Top+9b3Y/91cfnMhpnK8mipwgCDkmk8lUHeGT\nRJETBCHHZDL1HfkSRU4QhBzTQLTkBEHIx0R3VRCEfE2uIVd1hE9S3460IAjCNyBacoIg5JhclrOW\n3O7du/Hz80NTU5Phw4djbW3NuHHjyMjIwNTUlAULFqCtrc3u3btZt24dGhoauLq60qlTp2yPLYqc\nIAg5Jtf4+k5hfHw8y5YtY9u2bSQmJuLl5cXBgwfp2rUrzZs3x9PTk6CgINq2bcuyZcsICgpCS0uL\njh070qRJEwoVKpTl8UV3VRAElQoJCaFOnToYGBhgZmbGzJkzCQsLo3HjxgA0bNiQkJAQrly5gq2t\nLYaGhujo6ODg4MClS5eyPb5oyQmCkGMaOeiuPnr0iOTkZAYOHMirV68YNmwYSUlJaGtrA2BiYkJ0\ndDQxMTEYGxsr32dsbEx0dHS2xxdFThCEHNPI4RSSFy9e4O3tzePHj+nZsycfruX7qXV9P3e9X9Fd\nFQQhx2TZ/C8rJiYmVKtWDU1NTSwsLNDX10dfX5/k5GQAnj17hpmZGWZmZsTExCjf9/z5c8zMzLLN\nJoqcIAg5piHTyPKRFScnJ0JDQ1EoFMTHx5OYmEjdunU5ePAgAIcOHaJevXrY29sTHh7Oq1evSEhI\n4NKlS9SoUSPbbKK7KghCjuXk7Kq5uTlNmzbF1dUVgMmTJ2Nra4uHhwdbtmyhePHitG3bFi0tLcaM\nGYObmxsymYwhQ4ZgaGiY7fFl4kY235ZYain3iaWW8saXLLXU1MY1y9cPXtua0zhfLd+15O7du8eI\nESN48OABa9asoXr16qqO9FlMzUyY5TkRC8sSJLxJZM7UxVw8d5XBo3rTtFVDZBoa3PzrDjMn/sbr\nV28wMTVm6q9jKFPWggyFgt1BB1jju0kl2R8/fUbbXv0oWbyYcpuNdQV+8RhDXPwLJs9dQNTTZ+xa\n56eSfP/k6NlQVmzYnGnbg6jHHNu0joUrV3Pt1h00NDSo61CNoT93Ry5X/chO8KkzLPfzJzU1lYIF\nCzJ57GjKl7VixZp1/HHwCApJQcUK5Znq4Y6hgUGeZsuuS6pK+a7IbdmyhdevXxMaGoqurq6q43y2\nWZ4TOR0cxu9+W/lPnWp0/rk9ZkVNqV2vBq4t+pKamsbC5dPpO6Q7i+b44j55MH/fi2REv0noG+ix\nac9KroffJuzMRZXkNytiwnb/FZm2vXz1mn5jPHCsWYOop89UkutTGtetTeO6tZXPj5w+y+HTZ1m/\nbSdp6els8V5EekYGw6fPYu+x47Rp0liFaeFZdDSTZ/3KOt9llC1jyZZtO5g1/ze6/9SJQ0ePs3H1\nCnR1dRg/bSZrAzYxbGC/PM2Xk+5qblPfZF/p1atXFCtWDD09PbVeGeFD5sVMqWxbgU1rtwFwPuQy\nY4dMJ+LO38ya5ElKSiqSJHE+9E8srUoBUN7aSlnQEt4kcj38FuWty6jse/gnMhn8NmMK9evUUnWU\nLKWkpuK7cQvDfu5OxIOHONhUQUNDA20tLewqWhPxIFLVEdGUazJvxlTKlrEEoJq9HRH3/6aMZWlm\nTp6Avr4eGhoa2NtWIeL+33meLydnV3Nbvipy/fr1Y+fOnVy+fBlbW1vOnTuHt7c3TZs2xd7ensaN\nG+Pnl7nLtHPnTlq3bk21atVwcnJi6tSpylPXjx49wtrami1bttCgQQMmTpyYK7mtK5UjKvIJI8YP\nYPex3/HfsoSKVcpz+0YEt29EAGBgqM+PLRoQfOQMAGFnLvJjq4bI5XJMzUywsa/IuZDLuZLvcyQk\nJjJ62kza9xnA0AlTuP/gIUaGhliWKqmyTJ9r95Fj2FW0pmSxotSws+VE6DmSU1J5k5DIuSvh1Kxq\np+qImBgXxrH2+z8Wp0PCsKlciXJWZahc0Vq5/UxoGLZVKuV5PplMluVDlfJVkVu1ahVt2rShWrVq\nhIeHExISws6dO1m6dCmXLl1i3rx5+Pj4sHPnTgDCw8Px8PBg9OjRXL58mY0bN3Ls2DFWrlyZ6bi7\ndu1i8+bNzJ49O1dyGxY0oLy1FRfDruDSqAf7dhxm0YqZyOVvZ5HPXTqFo+e38/BBFHu2vT2t7rN4\nLTZ2FTn5524Ohmzl8B8nlAUxr+np6dKsYQPcB/UnyM+HWg7VGD1tJukZGSrJ8yUUCgUbd+2le9vW\nAHRs0ZT09Aya/exGs5/7UrKYOY7Vq6k4ZWZhFy4SsCWQsSOGZtq+au3vxMbF06VThzzPpCGTZflQ\npXxV5D6kUCjYuHEj/fr1w9raGrlcTo0aNejUqRNbt74902NjY0NISAgNGzYEwMLCgurVq3PlypVM\nx2revDlFixbNtb9Ib14lEBsTT/Dht620bZv3YlTQkNJWb1tB44fPpJ59a5ISk/l18WQAflk4niP7\nT+Bo25IGDm2pWdeBH1s2zJV82SlkZITHsEEUL2qOhoYG3Tu2Izb+BQ8fRakkz5cIv3UbPR0drCze\nDgN4rwuguLkZRwLWcGTDGpKTUwjYsVvFKd87duIUU2bNwWvBHGXXFWCJz0qOnjiJ7+KF6KlgLFou\nk2f5UKV8W+Ti4uJ48eIFM2fOxNbWVvkICAjgyZMnwNtCuH79epydnbGzs8PW1pZDhw6Rmpqa6VgW\nFha5mvVx1FP09HUzFVFJknD4jx1ly1sCkJqSyvZNe6lb/z8A1K1Xgz92HQHg1cvXhJw6T43a9rma\n81NevX5N1JOnmbYpFAo05eq7kOI7py9cou4HLbWwP6/i7FQXTU1NdAoUoF7NGlz667oKE74Xev4C\n8xd74bt4IVUqVVRu9/Fbw59Xw1ntvYTC2azIkVtEd1UFdHR0AFi0aBHh4eHKx7Vr1zh+/DgAPj4+\nrF+/nmnTpnHx4kXCw8Np3rz5R8fS0tLK1ax3bt4j+nks7Tu3AqBJiwa8evkakyLGuE8Zgpb228//\nwbkud27eA+Dve5H84OwIQIEC2tSs68DdW/dzNeen/HXrDgPHTST+xUsAdvxxgKJmppQo9vnzrFTl\nzv0HWJYsoXxuUbwYpy+8PaGTkaEg5NKflP1fK0+VkpKTmTp7Lp5zZmJlaancfv3mLfYcOMjSBXPQ\n19dTWT517q7muykk7xgYGFCkSBGuX79OkyZNlNufPXtG4cKF0dbW5vLly9SsWZN69eoBb1sf165d\nw9w87ydvjhk0lZkLJ+A2uCtxMfGMGTSNiDt/M85sKNsO+INMxrMnz5nuMR+AyaPnMOGXEXTq5oJM\nJuPMiXNs27Q3z3MD1KnhQKfWLekz0h2ZhgZmJiYsmDqRM+cusHiVP8nJKcTGx9O+zwDMTEzwXfCr\nSnL+k+jYWEwKv2/9jHbrxbwVfnQYNByAKuXL0atTexWley/45GniX7xkwvRZmbbb2VTm9es3dO87\nSLmtWFFzfBcvzNN8cjFPTjV+/vln/Pz8qFmzJjVr1uTOnTsMGTKEjh07MmjQICwsLDhz5gzx8fFk\nZGTg5eWFoaEhz58/Jz09PU+z3rvzgG5tBn60fdZkz3/c/+7t+7h1HpnbsT5bT9cO9HTNPOBdprSF\n2k8f2bAkczEwNy2C5+TxKkrzac1/dKb5j86qjvFdytdFzs3NjaSkJCZMmEBsbCxmZma0a9eOAQMG\nADBw4EAiIiJo2LAhJiYmDBs2jDZt2jB48GBat27NqlWrVPwdCML3QZ0nA4trV78xce1q7hPXruaN\nL7l2tUetAVm+/nvYiixfz035uiUnCELeUPXJhayIIicIQo6pc3dVFDlBEHJM1denZkUUOUEQckx0\nVwVByNdUfVVDVkSREwQhx8SYnCAI+ZoYkxMEIV/TUN8aJ4qcIAg5J7qrgiDka+LEgyAI+ZqGGJMT\nBCE/Ey05QRDyNbkan3lQ39FCQRCEb0C05L6x00eXqzrCFxnVZbGqI3yxxVtGqTrCF9HI5eXz1YFY\nGVgQhHxNTCERBCFfU+PzDqLICYKQc2IVEkEQ8rVvce1qcnIyrVq1YvDgwZw7d46//vqLQv+7j6yb\nmxsNGjRg9+7drFu3Dg0NDVxdXenUqVO2xxVFThCEHPsWU0h8fHwoWLCg8vno0aNp2LCh8nliYiLL\nli0jKCgILS0tOnbsSJMmTZSF8FPUd7RQEIR/jYiICO7evUuDBg0+uc+VK1ewtbXF0NAQHR0dHBwc\nuHTpUrbHFkVOEIQck2toZPnIzrx58xg/PvP9bgMCAujZsyejRo0iLi6OmJgYjI2Nla8bGxsTHR2d\n7bFFd1UQhBzLSXd1586dVK1alVKlSim3tWnThkKFClGpUiVWrlyJt7c31apVy/S+z72bqmjJCYKg\nUsHBwRw9ehRXV1cCAwNZvnw5kiRRqVIlABo1asTt27cxMzMjJiZG+b7nz59jZmaW7fFFS04QhBzT\nyMEVD4sXv7/qxsvLixIlSrBp0yZKlSpFqVKlCAsLo3z58tjb2zN58mRevXqFXC7n0qVLTJw4Mdvj\niyInCEKOfevr87t168bIkSPR1dVFT0+POXPmoKOjw5gxY3Bzc0MmkzFkyBAMDQ2zPZYocoIg5Ni3\nWmpp2LBhyq+3bdv20evNmjWjWbNmX3RMUeQEQcgxsZ6c8EWiY+OY7rmUyMdP0NfTY+ygvkQ9ecZv\nK/0pYlxYuZ9rq+a4tm6R5/lMihoza8s0oqPen77/+/oD/Gf9Tstezaj1Yw00ZBo8vBPJ7/M2kZSQ\njFxTTjf3n6hgXw6FQkHwjlMcCzqR59n/36nQc4yYNJ29G9ZQvKg5x06fZclKfxQKBdblrJg+djQG\n+nqqjglAWno6i72Ws37jZg7v3UlR87eD7r5+a9h34CCSJFGxQgWmTfLA0MAgT7Op83pyosipoeme\nS6lbw4Fu7Vy4cCWcwD37qVvDgQZ1ajF99LDsD5AHXkS/YGrXWZm2OTSoSo1GDszuu4DUpFT6Tu9F\n025N2LlyD006N0LfSI8pXWdSQLcAU9eOJ+LafR7cfKii7wCSkpPx8ltDwf+N60Q9ecrcJcvwW7yA\nUsWLsXD5Sk6FhtG8ccNsjpQ3ho/xwKZypUzbDh09xsEjR9m8bjW6urp4TJ7GmvUBDB88ME+zqXFD\nTkwhUTdPo2O4efceP/2vhVbD3pY5E9xVnOrzPH3wlDWzfyclMQVJkogIv0fxMkUBqNGwGid3nUGS\nJJITk7kYfJkaDatlc8TctWLdBlo4N0JPTxeAP44co1E9RyxKFEcmkzF2yAC1KXAAA9x6MWRA30zb\nrCwtmTVtMvr6+mhoaFDVzpa79+7neTYNmSzLhyp9siUXFBSU5Rs7duz4zcMIcOf+3xQ3N8Nr7e+c\nPncRk8KFGNO/DwC3791nwPgpxMTGUbVKZUb164WBvr5Kcurq6zB4Tj+KljYn9kkcW5Zu4/H9p5n2\nsaldmTtXIgAwtzAjOur9HKfoqBhs61TJ08wfunPvPmGXLrN+2WICd+8D4HbEfYqamTJo7ESePHvO\nf6rZM3pQP3R1dFSW80NV7Ww/2laurFWm56fOhlKjWtW8iqSk6kKWlU8WuYsXL2b5xrwqcvfu3WPu\n3LlcvXqVtLQ0KlasyMSJE6lSpQo9evTAxsYGbW1tAgMDSU1NpX79+vz666/o/O8Hc+XKlaxbt47E\nxETq16+Pk5MTkydP5tatWwDEx8czd+5cQkNDefHiBVZWVowcOZIffvgBgPHjx5OQkICOjg5Hjhxh\n9+7dmWZmf2uv3yRw98FD+nZxZVTf3uw4cJhxs+czbfQwfqhdk+7t2yDX0GC651I8V61h6sihuZbl\nU5ITUwg7fIFDG48S9ywe558aMnTuAKZ2n4UiQwFAi55NMTI24mhgMADaBbRJS01THiMtJQ1tHe08\nzw5vZ8r/utibcUMHoaX5/lfgdUICDy49wnfBHHR1dBg99Rf8N25hSJ+fVZLzS630X0tcXBxdO2e/\nMse39l0WuTlz5ii/VigUxMbGYmpqmiehPjRixAgqVqxIcHAwANOmTWPYsGEcO3YMgB07djBy5EiC\ng4O5d+8enTp1wsHBge7duxMWFsZvv/3GggULaNasGcePH+eXX37JdPyhQ4diaGjItm3bMDIyIjAw\nkMGDB3PgwAFlMTt//jwDBw5kzpw5yOXyXP1+DfT1MC5UkB/q1ASgbVNnlqxeRyEjIwZ076zcr5dr\ne4ZPnZmrWT4l4VUCmzwDlc8Pbz5Gq97NMS9lxpO/n9JuoAtV/lORxaO8SU1OBSAlOQUt7ffLgGvr\naJOSlJLn2QG27d2PVWkLqtlmbkka6OthV7kixoXfrmrR0aUlazcFfhdFbrG3D2fDzrHCezF6urp5\n/vnqfHY12zG5kJAQnJ2d6dGjBwC//vqrsuDkhU2bNjFz5kx0dHTQ0dGhRYsWREVFKS/MLVmyJJ07\nd0ZbW5uKFStibW2tbKXt378fa2trXFxc0NbWpmnTptSuXVt57Js3b3LhwgU8PDwoUqQI2tradOvW\nDWtr60xzdGQyGT179kRTUzPX/2MWMzMlMSkJhUKh/GwNmYz4l6+If/lSuV9GhgJNTdWcN9Iz1KVI\nMZNM2zQ0NMhIz6B1nxaUs7ViwbAlvHmZoHz96YNnmJV8/0fSrKQpT/7O3L3NKyfOhhJ8NpQmHbvR\npGM3nkXH0H3wCKKePOVNQqJyP7mGBhpqvKz3O8tX+nH56lXW+HpTOJtlh3KLTJb1Q5Wy/S+4aNEi\ntm7dqmzFDRw4kOXL8+5mLZcvX6ZXr17UqFEDW1tbhgwZAkBKyttWQOnSpTPtr6urS1JSEgBPnz7F\nwsIi0+tVq74fr7h37x4ALi4u2NraKh+3b9/m8ePHyv1KlCiRZz/s5SxLY2pszM6DRwA4cuoshgYG\nhF66zOylPqSnp5ORkcGWPX/gWKN6nmT6f5aVSjNm6TAMCr2dplDPxZG4Z3Ho6OtQp1lNvMf5kpKY\nuZV24dhlGnX8AZmGjIImRvzHuTrnj2a/TE5u8JrzC0e3beJw0AYOB23A3LQIAcuXMGHEUA4dP8mz\n6BgyMjLYuf8Qtarn/fjWl/jrxk127zuAt+cC9FU0Pgtvp5Bk9VClbJsCenp6FClSRPnc2NgYrTy6\n+9D9+/cZNGgQPXr0wNfXl0KFCnHq1Cn69n1/himrlpVCofgo64f7FyhQAIDTp09nWqzv/+XV9wtv\n882dOJYZi7xYF7gD40JGzJ3gTplSJZm7fCWug0Ygk8mwq2TNCLeeeZbrQ9fP3eT4jlOM9xmNQlLw\nIvolPpP8cHZtiJ6hLhNWvT8bHPc0nsWjl3F063GKljZn1qapZGRksHfNfh7djVJJ/k+xq1yRAT93\no88IdzQ1NalmW4XenV1VHQuAmNg4eg8YrHzeZ+AQ5HI51ava8/rNa7r2ev87UbxYUVZ4fX93Ycst\n2RY5HR0dzp07B8DLly/Zt2+fsjjktuvXr5OWlsaAAQOUq39euXLls99vamqq7Lq+c/nyZeXXlpaW\nys+pU6eOcntkZCQlS5ZU2TiDlUUp1i2a/9F2dZkjB3Bo41EObTyaaVvAgs0ELNj8j/tnZChYP3dj\nXkT7Yvs2rlV+3cmlJZ1cWqouzCcUMTFmT9A//9tOnzwhj9N8TEONJwNn2webNm0aq1evJjw8nCZN\nmnDq1KmPBu9zy7uB/4sXL5KSksL+/fs5f/48AE+ePMn2/c7OzoSHh3P48GHS0tI4dOiQ8v0AZcuW\nxcnJiXnz5vHgwQMyMjI4fPgwLVu2zPbssiAI7+V00czclG1LrlixYqxYsSIvsnzEzs6OgQMHMnHi\nRBQKBc7Oznh7ezNgwAD69etHUlISrVu3/uT7GzRogJubm3I5FmdnZ9zc3DKdOV6wYAG//vornTp1\nIi0tjdKlSzNv3jxq1KiR69+fIAi5TyZls7zm+fPnmTt3LhEREchkMipUqMC4ceOoXl01g95fKiUl\nJVP32svLi+3bt3P8+PFc+bxXd//KlePmljE/+6g6whdbvGWUqiN8ES0j1ZzxzCltI5Psd/qfDX09\ns3y9m9/onMb5atm2I3/55Rfc3d0JCwsjJCSE4cOHM2PGjLzIlmNXr17F3t6e/fv3o1AouHv3Ltu2\nbaNx48aqjiYI+Yo6TyHJtrtqYmKSaVDe0dGR4sWL52qob8XOzo5ffvmFxYsXM378eAoXLoyzszOj\nRn1fLQFBUHff5RUPkZGRANja2uLv70/dunXR0NAgJCSEypUr51nAnHJ1dcXVVT2mAQhCfqXGNe7T\nRe7nn39GJpMp74gTEBCgfE0mkzF8+PDcTycIwndBnaeQfLLIvbs29J98zg1dBUH491Dna1ezHZN7\n8+YNu3btIj4+HoC0tDS2bdvG6dOncz2cIAjfBzWucdmfXR05ciS3bt1i+/btJCQkcPz4caZPn54H\n0QRB+F6o87Wr2Ra5lJQUfvnlF0qUKIGHhwfr169n//79eZFNEITvhEwmy/KhStl2V9PS0khMTESh\nUBAfH0/hwoWVZ14FQRBAvbur2Ra5Nm3asHXrVjp16kSLFi0wNjb+aPkiQRD+3VTdWstKtkWuS5cu\nyq/r1KlDbGzsdzVPThCE3PddTiFZsmTJJ990+PBhRowYkSuBBEH4/qhxQ+7TRS6372UgCEL+8V12\nV4cOzfu7QOULWS/qona8dk9WdYQv5jPAT9URvsiQ1Xl7o2dVUPU0kayo5k4ogiDkK2rckBNFThCE\nnPsuu6sfio+P59GjR9ja2qJQKL6L27QJgpB3clLjkpKSGD9+PLGxsaSkpDB48GAqVqzIuHHjyMjI\nwNTUlAULFqCtrc3u3btZt24dGhoauLq60qlT9jfSzrbI7d27l6VLl6Ktrc3evXuZOXMmlStX/qyD\nC4Lw76Ah//oqd/z4cWxsbOjXrx9RUVH06dMHBwcHunbtSvPmzfH09CQoKIi2bduybNkygoKC0NLS\nomPHjjRp0kR5k6tPZssuwJo1a9i1axeFCxcGwMPDg61bt371NyQIQv6Tk8u6WrRoQb9+/YC3N6gy\nNzcnLCxMuYJ3w4YNCQkJ4cqVK9ja2mJoaIiOjg4ODg6ftSJSti05Q0NDdHV1lc91dHTy9D6kgiCo\nv28xJNe5c2eePn2Kr68vvXv3RltbG3i7Onl0dDQxMTEYGxsr9zc2NiY6Ojrb42Zb5AoXLsyOHTtI\nSUnhr7/+4o8//sj0QYIgCN/iiofNmzdz48YNxo4dy4f31/rUvbayuQfX+2zZ7TBjxgzCw8NJSEhg\n8uTJpKSkMGvWrM+MLQiCkLVr164p76NcqVIlMjIy0NfXJzk5GYBnz55hZmaGmZkZMTExyvc9f/4c\nMzOzbI+fbUvOyMiIqVOnfm1+QRD+BWQ5mHFx4cIFoqKimDRpEjExMSQmJlKvXj0OHjxImzZtOHTo\nEPXq1cPe3p7Jkyfz6tUr5HI5ly5dUt5TOSvZFrkffvjhHwcOg4ODv+obEgQh/8nJmFznzp2ZNGkS\nXbt2JTk5malTp2JjY4OHhwdbtmyhePHitG3bFi0tLcaMGYObmxsymYwhQ4ZgaGiY7fGzLXIbN25U\nfp2WlkZISAgpKSlf/x0JgpDvyHIwJqejo8Nvv/320fY1a9Z8tK1Zs2Y0a9bsi46fbZErUaJEpueW\nlpa4ubnRq1evL/ogQRDyr+9yqaV3QkJCMj1/+vQpDx8+zLVAgiB8f9T4qq7si9zy5cuVX8tkMgwM\nDJgxY0auhhIE4fvyXV+7On78eKpUqZIXWYT/iY6NY/oiLyIfP0FfT5exA/tiV6kiXmt+58yFi6Sk\npOLaqjk9OrRVdVSl4FNnWO7nT2pqKgULFmTy2NGUL2vFijXr+OPgERSSgooVyjPVwx1DAwOVZi3j\nUI62EzuzepAXr2Nf8UOvHyltXwaZTEbktb855ncASSFRqJgxjfu3wMDYAEW6gou7Q7l+4qpKs6el\np7PY24f1GzdzeM8OippnnkKxcIk3h48d5+CubXmaS41rXPbz5ObNm5cXObJ19uxZrK2tefTokaqj\n5Lrpi7yoW70au/19GdO/D4F797Pz4GH+un2HDUt/Y5O3J7sPH+PyteuqjgrAs+hoJs/6lTnTp7Bz\n0++0aNKYWfN/4/CxYA4dPc7G1SvYtel3ZMhYG7BJpVk1tTVx6taIpNeJADi0qoVxCRN+H7OS9aNX\nYFLKjCqNqgLQdEhr7oTeYN0IX7bP3Ej9n50pVEy1E+GHu3ugp6f7j6/dun2HYydO5nGit2RyWZYP\nVcq2yBUvXpwePXqwcOFClixZonwIueNpdAw3797jp9YtAKhhZ8uc8e6EXb5K0x/qUUBbGwN9fVo7\nN+LY2VAVp31LU67JvBlTKVvGEoBq9nZE3P+bMpalmTl5Avr6emhoaGBvW4WI+3+rMip1XOtz42Q4\nqUmpADy6/pDjqw+iSFegSFfw9G4UJiWLAGBiYUZk+H0AEl68If5JrPI1VRnQpzdD+vf9aLtCoWDm\nvIUMG9hfBanU+5aE2Ra5kiVLUqtWLXR0dJDL5cqHkDvu3P+b4uZmeK0NoMOAYfQfP4VbEfeQyd7+\nIL+jq6vDoydPVZj0PRPjwjjWrqV8fjokDJvKlShnVYbKFa2V28+EhmFbpZIqIgJgYmGKhb0Vl/aG\nKbc9u/uY+MexwNtpEKXtrHh65zEAkeH3sXasAjIoXNwYI9NCPLkTpZLs71S1s/nH7YHbd1G+XFns\nbFQztCSTZf1QpU+Oye3evRsXF5ccLYO+b98+fH19efToEVpaWtSsWZMpU6Zgbm7O6tWr2bp1K8+f\nP8fIyIi2bdsycuRIZdU/cOC8Y6W0AAAgAElEQVQAS5Ys4cmTJ9jY2NCqVatMx27UqBE9e/bk3r17\nHDhwAJlMRsuWLZkyZYryGPv372fVqlXcv38fHR0dnJ2d8fDwwOB/Y0K///47AQEBPHv2DF1dXRo0\naMCkSZMwMDAgMjKS2bNnc/nyZVJTUyldujRDhw7F2dn5q/89PsfrhATuPnhI3y6dGNW3FzsOHGbc\nrwvo1rY1Ow8dpUXDH8hQKNh/7AQ6OgVyNcvXCLtwkYAtgazyWpRp+6q1vxMbF0+XTh1UlAyc+7d4\n22rLUPzj6437Ned13Ctuh7wdBghecxjXmT2p1qIm2noFOO5/kMQXCXkZ+bPExMQSsHkLAf4refNG\nNfnUeQrJJ1tyQUFBOTrws2fPGDt2LO7u7ly6dImDBw8CMH/+fA4ePMiiRYv47bffuHz5MsuWLWPt\n2rVs374dgEePHjF69Gg6dOjAuXPnmDRpEmvXrv3oM/z8/Khfvz5nz57lt99+Y8OGDcorMc6ePYuH\nhweDBw/mwoULbNmyhWvXrjF79mwA/vzzT+bPn8+iRYv4888/2blzJ/fv32flypUATJ8+nUKFChEc\nHMz58+fp3bs3Y8eO5cWLFzn6d8mOgZ4exoUK8kPtmgC0berMy9dvqGZTmVrV7Og1Zjwevy6gZjU7\nDPX1czXLlzp24hRTZs3Ba8EcZdcVYInPSo6eOInv4oXo6f7zeFJus23iQOyjGB7f/PjG6DINGU2H\nuWBQxIg9C4KQFG8v/G49tiMhW07g0/s3Vg/yooZLbYpVKPHR+1Vt/uKlDOjbm4JGRqoLocZNuVxb\n/vzNmzdkZGSgq6uLTCajcOHCeHl5IZPJyMjI4NSpU8o16mxsbChfvjxXrlyhQ4cOHDhwAENDQ3r3\n7o1cLqdSpUp06tSJ+fPnZ/qM6tWrK1tWTk5OGBsbc+vWLRo2bMiGDRto0qSJ8nULCwuGDRvG8OHD\nmTZtGq9evQLezrYGMDc3Z9OmTcpW4OvXrylatCja2trI5XLatGmDi4tLro8vFDMzJTEpSbkCs0wm\nQ0MmQ0tLixF9fmZEn58BWLVpK+UsS+dqli8Rev4C8xd74bt4IVaWlsrtPn5r+PNqOKu9l6Cvr6ey\nfGX/UwHzssWwql4eAF0jPbrM7cM+z+1U/sEOTW1Nds/dqmzl6RjqYl62GDdPXQPgTdxrHt96RPGK\npXhyW7Vd1v934vQZzl24xMLF3igUCl6+ekWDZq05tHubcrmi3JaTKx5y2yeL3OXLl2nQoMFH2yVJ\nQiaTZXvtatmyZenZsye9evWiQoUK1K5dm+bNm2Nvb09qaipeXl4cPXqUuLg44O0lY+XKlQPeLpxX\nvHjxTGN/7177UOnSmX/JdXV1SUpKAuDevXs8ePCAQ4cOZdpHoVDw7NkzateujbOzM61atcLOzo7a\ntWvTqlUr5eeMHDkSd3d36tWrR506dahfvz7NmzfP9R+acpalMTU2ZuehI7Rv9iNHTp/F0ECfa7du\ns3LDFmaNHUls/Av2HjmO90z1WDghKTmZqbPnsnju7EwF7vrNW+w5cJAta/1UWuAAdv66OdPzPsuH\nEjTtd8ysimJcqghbJ6/L1I1NfpNE4ssErGqU527YLQro61DcuqSy6KmTsOAjyq+jHj+hz6ChYgrJ\nBz5Z5CpXroynp2eODj5p0iT69u3L6dOnOXnyJN26dcPNzY3nz59z+vRpli1bRpUqVZDL5fz000/K\n96Wmpn7UYvqntaOyalXp6OjQtWtXJk/+9C33Fi1aRGRkJKdOnSI4OJiVK1cydepUOnfuTN26dQkO\nDiYsLEzZHfb19SUwMFA5ppcbZDIZcye4M2ORN+sCd2BcqCBzJ7hjWbIEJ0LP067fEOQaGgzt1Z1S\nxYvlWo4vEXzyNPEvXjJheuYluOxsKvP69Ru69x2k3FasqDm+ixfmdcRPsm3igJFpQXp4vj8r+fjW\nIw4v38vehduo37Mxjl0bIZPBX8ev8vflCJVljYmNo/fAIcrnfQYNRS6X47dsKeZmpirLBeo9JvfJ\nIqetrf3RdatfQqFQ8OrVK8zNzenQoQMdOnQgMDCQOXPmYGZmRtOmTbGzswMgISGBu3fvUqZMGQCK\nFi3K0aNHla1GgNu3b3/R51taWnLjxo1M2169eoVCoaBQoUKkp6eTmJhIqVKl6Nq1K127dmXx4sVs\n2LCBzp07ExcXh7GxMfXq1aNevXoMGTKEevXqcfbsWX788cev/nf5HFYWpVi36OP5iQsmjcvVz/1a\nzX90pvmPuXtC5lvzH+wNwI5Zn563F3XjIZsmfHyRuKoUMTFmT2DW8wxLFC+W5604dffJEw/vCtDX\n2rt3L61ateLq1atIkkRCQgLXrl3DysoKCwsLrl+/TmJiIlFRUUyePJnixYvz5MkTJEmiUaNGxMbG\nsn79elJTU7l27Ro7d+78os/v2bMnFy9eZMOGDSQnJxMdHY27uzujRo0CYNWqVXTt2pWIiLd/mV+8\neMHt27exsrIiMTGRH3/8EX9/f5L+Nz529epVUlNTsfygOyYIwlsyuUaWD1X6ZEtu7NixOTpw69at\niYqKYuTIkcTExKCnp0f16tXx9PQkNTUVDw8P6tSpQ8mSJfHw8CApKYkJEybQr18//Pz8mD9/PsuX\nL8fT05PKlSszaNAg3N3dP/vzq1Wrpuxizp07FyMjI5ycnPDw8ADAzc2N+Ph4evXqxcuXLzEyMqJu\n3bqMHz8ePT09fH19WbhwofJkiYWFBfPmzaNChQo5+ncRhPxInU88yKTPXShd+Cyv7qjfwHRWtI1V\nO4P/a/gM8FN1hC8yZPVAVUf4KtoFP/9n47rflixfr9z3pyxfz025NoVEEIR/j5zcdzW3iSInCEKO\nqfr61KyIIicIQs6pb40TRU4QhJwTLTlBEPI1Va8ZlxVR5ARByDHRkhMEIV8TRU4QhHxNdFcFQcjf\nREtOEIT8TI1rnChygiDknBiTEwQhX1P1SiNZEUVOEIScU9+GnChy35p2YdXefPhLaWhpqTrCF/ve\nVvUY326uqiN8Fc9jn7+Cszp3V9W3jSkIwnfjWyyaefv2bZydnQkICABg/PjxtG7dmh49etCjRw/l\nfWV2795Nhw4d6NSpE4GBgdkeV7TkBEHIsZy25BITE5k5cyZ16tTJtH306NE0bNgw037Lli0jKCgI\nLS0tOnbsSJMmTShUqNAnjy1acoIg5Jwsm0c2tLW1WbVqFWZmZlnud+XKFWxtbTE0NERHRwcHBwcu\nXbqU5XtEkRMEIcdkGrIsH9nR1NRU3gP5QwEBAfTs2ZNRo0YRFxdHTEwMxsbvx72NjY2Jjo7O+thf\n/u0IgiBkJtP49u2lNm3aUKhQISpVqsTKlSvx9vamWrVqmfb5nLs3iJacIAg5pyHL+vEV6tSpQ6VK\nlQBo1KgRt2/fxszMjJiYGOU+z58/z7aLK4qcIAg5JpPJsnx8jWHDhhEZGQlAWFgY5cuXx97envDw\ncF69ekVCQgKXLl2iRo0aWR5HdFcFQcixnN6S8Nq1a8ybN4+oqCg0NTU5ePAg3bt3Z+TIkejq6qKn\np8ecOXPQ0dFhzJgxuLm5IZPJGDJkCIaGhllnE7ck/LaSYx6rOsIX0dAuoOoIX+47+5H9N0wGfhp8\nLMvXizZolNM4X0205ARByDFx7aogCPmbGl/WJYqcIAg5ps7XrooiJwhCjonuqiAI+Zsat+TUt/zm\nAltbW7Zv367qGNkKPnUG15/70rbrz/w8aBh37t0H4NKVcNp3703LTt3oO2w0z6NjsjlS3klLT2fB\noqXY/qcuT589V26PjYuj35ARtGjXSYXp/llaejoLFnthW9MxU+Z3Fi7xpmmbDipI9rFKtSrheWwh\nhc0Lo62jTedxPzF+3TgmBUygurODcj8zCzMGew7CY81YxvqNwbaeTZ7ky+llXbnpX1XkwsPDad++\nvapjZOlZdDSTZ89lzvTJ7Ny4jhZNGjNr/m+8SUhg3JQZTB8/ln2BG6hb6z/sP5L1afu8NHyMB3p6\nepm2vXz5it4DhlC+nJWKUmVtuLsHenq6//jardt3OHbiZB4n+mdaBbRo1a8FCS8TAPixRxO0dbSZ\n12sB3iOX06p/S4yLvr2e8+dpPTl/6ALzei8gYPYGuo7vgo7+x9eEfnMyWdYPFfpXFbnvgaamJvOm\nT6FsGUsAqtnZEnH/b46fOkNF6/LY2VQGoE/3LvzcxVV1Qf/PALdeDBnQN/NGGSxZMJeG9eupJFN2\nBvTpzZD+fT/arlAomDlvIcMG9ldBqo81/flHLhy+SEpSCgAVapTn/MHzSJLEy5iXXDvzFzaOVZBp\nyDj8+2EuHroIwJP7T8lIy1AWwNwk09DI8qFK/6oiZ21trVxkb/PmzbRu3ZqqVavi5OTErFmzSElJ\n4c2bN1SrVo2tW7dmeu/+/fuxtbXlxYsXuZrRpHBhHGvXVD4/HXoOm8qVuH03gsIFCzJywhRad+7B\nuKm/EP/iZa5m+RJV7Ww/2lbQyIgylqVVkObzVLX7565c4PZdlC9XFjubKnmc6GPFyhSlQvUKnAh6\n36qUpMwXxKckpVCkRBEkhcSfwVdQKBQAWFS0ACD6UdardHwLoruqZrZv3868efMYP348Fy5cwM/P\nj6NHjzJnzhwMDAxo0aIF27Zty/Seffv2Zbs437cWduEiAVsCGTt8CK9fvyHk3AVGDxnI9oA1aGtp\nMX+Jd55l+beIiYklYPMWRg5RjyXWO47qwA6vHSgyFMptty/exqlNXTS1NClkVghbJxs0tTKfQyxk\nWpDuk7uy3WsHaSlpuR9UdFfVS0BAAO3atcPR0RFNTU0qVqxIjx492LVrFwqFgp9++ok///yTu3fv\nAvD69WtOnDhBhw55Nwh97ORppsyeh9f8OZQtY4mBgT41azhgUbIEWpqadHPtQMi5C3mW599i/uKl\nDOjbm4JGRqqOQp1WtXn24Bn3r/2dafvh34/wMuYl7n5j6DSqAzfP3SQpIUn5umkpUwYvGsTRjce4\ndPRy3oRV4yL3r5xC8vDhQzp27JhpW9myZUlMTCQmJgY7OzsqVqxIUFAQ48eP5/Dhw5iamn60NHNu\nCT1/kfmLvfFdtACr/3X3ihU152FklHIfDQ0N5Go8N+l7deL0Gc5duMTCxd4oFApevnpFg2atObR7\nG9ra2nmaxcaxCiUrlGJ6nbfjsAYFDRjlM4L1v/zOloXv723w01hXIq/cA6BgESP6z+3H3pV7uXLi\nap5lFfPk1ExKSspHi+29e/5u5rarqyve3t64u7uzb98+2rdvj0YeDKAmJScz9dd5LJ4zU1ngABrV\nc2L5qjXcibhH+bJWbNu9l1o1qud6nn+bsOAjyq+jHj+hz6ChHNy1LYt35J5VE1Znej5540SWjfKh\nWsOqVK5dmd2+ezAvbU4Fh/LsWr4bgA4jO3By28k8LXAAMpkocmrF0tKSW7duZdp2+/ZtjIyMKFKk\nCAAuLi4sWLCAvXv3EhYWxi+//JIn2YJPnSH+xQsmzJidabv/ssXMmDiOUROmIJPJKGtVhqnjxuRJ\npuzExMbRe8Bg5fM+A4cgl8vp26snfmvXk5ycQkxsLK07dsbc1BQ/Hy8Vpn0rJjaO3gOHKJ/3GTQU\nuVyO37KlmJuZqjBZ9s4dPE/Pyd2ZFDCBtNQ0Ns7dTHJCMkYmRtjUrYJZKTPqutRV7r9nxV6uh1zP\n3VBqPBn4X7XUkrW1NbNmzSItLY158+bh4+NDrVq1uHHjBoMGDaJ169aMGzdOuf+ECRM4dOgQ9vb2\n+Pv7f9ZniKWW8sB39iP7b1hq6eXt8CxfL1jh47PveeVf2ZLr0qULiYmJzJo1iydPnmBmZkb37t1x\nc3PLtF+nTp3Yvn37R+N3giB8P/6VRU4mk9G3b1/69v14IuiH3q0f36RJkzxKJgjfJ1VP+M3Kv6bI\n3bv39uzT585zi4iIYN68eQwdOhQtLa3cjCYI3z9x4kG1du3axaRJk6hZsyaOjo7Z7u/m5sa1a9dw\ndXXF1VV9Lp0SBHWl6qsasvKvKHJt2rShTZs2n73/6tWrs99JEAQl0V0VBCF/E91VQRDyM9FdFQQh\nf1PjycCiyAmCkGMyDbmqI3ySKHKCIOSc6K4KgpCfiVsSCoKQr4nuqiAI+ZuYQiIIQn4mppAIgpC/\nqfGY3L9qPTlBEP591LcjLQiC8A2IIicIQr4mipwgCPmaKHKCIORrosgJgpCviSInCEK+JoqcIAj5\nmihygiDka6LICYKQr4kip0LiYhNByH2iyKmQTCYjODiYtWvXqjrKF/Pz8yMyMlLVMfItHx8ffHx8\nVB0jXxBFTgXeteBiY2NZsmQJlStXVnGiL3PixAm2bNlC4cKFVR0lSx+2lL+nVnN6ejovX77kzJkz\nxMTEqDrOd08UORWQyWScP3+efv36YWlpiZ2dHfB9/CIeOHAAf39/GjVqhIGBASkpKaqO9I8kSUIm\nk3H69Glmz55N9+7defHihapjfRZNTU2cnJx48OABd+/eBUChUKg41fdLFDkVKVasGAkJCRw4cICw\nsDDlL6W6u3nzJk+fPuXYsWO8fv2aAgUKqOUvoEwm48CBA4wYMQJjY2Pq169PoUKFVB3rIx/+YUtP\nT1d+7eTkhJOTE4sXLyY5ORkNNb55s7qTT58+fbqqQ/wb/H8RMzIyolGjRpw8eZLQ0FAcHBwoUqSI\nChN+npo1a6KlpUVoaChXrlyhbt266OnpoVAo1KpI//XXX0yYMAFPT0/atWtH1apVSUpK4sCBA2hp\naaGrq4umpuqXU3z3b7Zq1SpOnTqFtrY2xYsXB8DU1JTQ0FAqVaqEubm52v0bfy9EkcsD7wrchQsX\n2L59O3/99RcymYwKFSrQoEEDtm/fzsmTJ6lWrZpaFrp9+/YRHBzM5cuXMTMzw9HREW1tbUJCQvjz\nzz+pVasWenp6ZGRkqE2L4+bNm0RHRzNgwABev37NsmXL8Pb2ZseOHezatYtKlSphaWmpsnzvfibe\n/f+ePXu4ePEi69at48mTJygUCurUqcP+/fuJiIjA2dlZFLivJIpcHnjXdRo7diza2tqcOHGCCxcu\nUKxYMWxsbGjYsCFBQUGcPXsWGxsbTE1NVR1Zae7cufj7+2NgYMDx48fZu3cvhoaGdOzYkaSkJM6d\nO8fVq1f5z3/+g76+vqrjKj148ICNGzdy9epV5syZg56eHg0aNMDHx4crV65w584dmjZtqpJsCoVC\n+cfg+fPnaGho0LhxY1xdXbGwsODYsWMcOHCAs2fPUqtWLfbv30/16tUxMzNTSd7vniTkuhs3bkiO\njo5ScHCwJEmStGPHDsnOzk5q166dcltkZKTk6Ogo/fTTT1JKSooq4yoFBgZKTZs2lWJjYyVJkqTN\nmzdLVapUka5cuSJJkiQpFAppzZo1kqOjo+Tp6amynAqFQpIkSQoPD5f27Nkj7du3T5IkSVq7dq20\nYMECyd/fX3r9+rWUlpYmSZIkeXp6SgsXLpQyMjLyPOuHn+nl5SV16tRJcnZ2llxdXaXw8HBJkiQp\nLi5Oun79ujRw4EDJxcVFsra2ltasWSNJ0vvvVfh8osjlgT179kg9evSQJEmSHj58KNWpU0caPXq0\n1LVrV8nFxUW6ePGidPPmTenx48dSZGSkitO+N2fOHGnevHmSJL0tzPb29sqiHBERISUnJ0sZGRnS\nnj17pPT0dJVkfPdL/8cff0iOjo6Sq6urtGTJkk/uHxAQINWuXVu6c+dOXkWUJEn66A/XwoULpfr1\n60uhoaHSunXrpL59+0r29vbSyZMnM+13584dydPTU2rQoIHyj43wZUSRyyUf/sXdsWOHNHbsWCk+\nPl5q3LixtGjRIkmSJCkoKEiqWbOmVLVqValSpUpSVFSUquJ+JDU1VRo2bJg0e/Zs6cSJE5KdnZ2y\nwMXFxUmOjo7Szp07M71HVYXu9OnTUvXq1aVTp05l2h4VFSUlJydLkiRJPXv2lIYNGyY5OjpK165d\ny9N8bm5u0pkzZ5TPb9y4IbVv3156+PChctuTJ0+k0aNHS1WrVpUePHiQ6f2PHj2SXFxcpNu3b+dZ\n5vxE9aeX8hnpfwPJaWlpaGtrA9C2bVvatm3Lrl27MDc3Z+TIkcDbM6wuLi40aNCAUqVKKc+qqZK/\nvz8NGjTAysqKpk2bMmbMGNavX8/GjRtxcHAAoHDhwpQrV44yZcpkeq9cnvc3GM7IyOD06dMMHjwY\nJycnYmNjOXz4MHv27CE+Pp6SJUvi6emJk5MTpUqVYuzYsZQqVSpPM1aqVIm6desqn6empvL06dNM\nJ2mKFi3KiBEjuH//Pps2bWLs2LFIkoRcLqdEiRIAxMTEUL58+TzNnh+IIvcNvStwJ0+e5MCBA8TG\nxuLg4EDLli0pWbIkCQkJ3L17lytXrmBvb8/Dhw8pUaIEjo6Oqo4OwKNHj9ixYwd+fn5s3bqVli1b\ncuPGDfz9/YmKisLU1JRixYoxceJEEhMTqVKlisqyvvu3lsvlpKSksHr1akqVKsWyZcsoVqwY1tbW\n2Nrasnr1akJCQujXr5/KMo4ZMwaAFStWYGJiQtmyZZXTQ1xcXNDS0kKSJCwsLChatGimExMAO3fu\nJCYmRqVng79nosjl0IULF6hcuTJ6enrIZDIOHjzI+PHjGThwIAYGBhw6dIht27axa9cuLCwsKFas\nGNOmTcPKyorjx48TGBio6m8BgKVLl3L37l0KFy7MnTt3+Omnn9i6dSvu7u5oaWkxceJEjI2NMTIy\nQktLiy1btiCXyz/6hcxt7wqHQqFQthx79+5NZGQkv/32G46OjnTp0oVy5coBsH///jzL9v/+f8rH\nqVOnuHHjBlOmTMHIyIiAgABMTEyoV6+e8nvR0tKiaNGiwPvvVVtbmzVr1lCsWLE8/x7yA3Hf1Rzw\n9vYmKCiI48ePI5PJiIyMZOjQoYwYMYJGjRrx+PFjWrduzYABA+jfvz8AwcHBhIaGEh0dTf/+/bG2\ntlbxd/G2i7pixQqCgoLQ19cnNDSU1atXExUVRVBQECVLluSvv/7i8ePHaGtr4+TkhFwuJz09PU8n\n1L77pT9z5gx79+4lPT2dUqVKMXz4cODtdIwPp1ls2LABf39/1q1bR8mSJfMs5/+TPpgIPnDgQG7f\nvs3gwYPx8/NDU1MTR0dH6taty+7du7lz5w7bt29HU1Pzu7kKRt2JIveV0tPTcXd3JykpiRUrVhAd\nHU1ERAQzZ85k3759PHr0iI4dO9K5c2dGjhzJixcvCAkJoVmzZshkMjIyMlQyhvX/kpOTcXd3p1y5\ncsqxQoCrV68yfvx4Xr16xaZNmz4ax1JV/kOHDuHh4UHnzp3R1dVlzZo12NnZsWLFCnR0dEhNTWXp\n0qVIksSWLVtYt26dSrvV73zY4u3bty8PHjxgwIABhIaG8ujRI9LS0rC0tGTu3LloaWmpzc9HfqAe\n09O/QzKZjMqVK3Pv3j0GDhxIs2bNSEhIAGDv3r24urrSrVs3ZeFYsWIFgYGByr/M6vIDrKOjg46O\nDgcPHsx0sb2dnR0uLi7ExMTQqVMnHjx4ALwtbqCa/JGRkSxatIj58+fj4eFBu3bt0NTUpE6dOujo\n6ABw/vx5IiIiePnyJZs2bVKLAgegoaGhvMbXz8+P0qVLs2zZMpycnPD29mb9+vUsWLAALS0t0tPT\n1ebnIz8QRe4ryeVy+vfvT5kyZTh58iT16tWjcePGGBsb4+7ujrOzM8OGDVPur6OjQ+nSpcnIyFCL\n1UbevHmj/Lp169ZIksTChQszba9WrRrdunWjatWqDB8+nLi4OJX+8sXGxqKjo0OTJk2ULeUePXow\ncOBA3rx5w549e3B0dMTHx4cZM2ao3ZnI/y90FSpUYPbs2Zw9exa5XK58XR2uqc1PxGVdOZCcnMya\nNWuwtbUlIiKCxMRERo0axYULF5TjVxkZGezevRt/f39mzJhBkSJFVD7O4uvry6JFizh9+jS6uro0\naNCAmJgYTp06xaNHj7CxsUFXV5eVK1diamrKDz/8wB9//IG9vT0WFhYqy/3ixQt27txJgQIFGDt2\nLF27dlWOxy1btoxTp07h4uICvG1pq/rf+Z+8O2kik8lo3bo1ly9fZvPmzZQpU0ZZlNUx9/dM/MnI\nAR0dHYKCgtDW1mbmzJls27aN9PR01q5dy+jRo/Hy8kIul2NgYMC6devUomXh6+tLQEAAffv2ZcuW\nLSxdupSEhARGjBiBXC7njz/+YMeOHZQoUYLk5GQOHToEwJIlS/K0Bfpu0D0mJgYDAwMKFChA8eLF\nKVGiBDNmzKBVq1aZWsrJycmZxg3VuVC8a7FpaGjg4+NDv379WLNmDU2bNlWbBQ7yE1HkcujdhN8B\nAwYgk8nYuXMn2tra+Pr6EhMTQ3x8PCYmJhgbG6s46fuVOd6dSGjUqBEzZ85k1apVSJLEsGHDaNq0\nKcHBwRQsWJCOHTsC8Pvvv5OSkkKFChXyLKtMJuPw4cMsX76cxMRE2rdvT/fu3ZkwYYLyRM7mzZup\nU6cOBw8eZPv27WzatCnP8uXUh4XO3t6ewMBA3rx5g5GRkaqj5Tvi7Oo3FBMTg6+vL2fPnsXR0ZFJ\nkyapOpLS7t272bx5M3K5HC8vL+UCkg8ePGDWrFnExMTQo0cP2rdvr3zP4MGDAbh48SL+/v65Poj/\n4ZSJCxcuMHDgQEaNGsWtW7c4cOAArVq1YvTo0URGRrJs2TJCQkIwNzdHJpMxf/58tTnJ8CVSU1NZ\ntWoVjRo1olKlSqqOky+JMblvSE9PT3nG9fr16zRu3Bg9PT1VxwLeFpDTp08THh6Oubk5tra2ABQq\nVAh7e3suX77MH3/8QcmSJbGysuLFixfcvXsXGxsbRo8enautuPT0dGU37V0X9ejRo7Rr14727dvT\nsGFD5HI5W7duJTo6GmdnZ9q1a4eLiwsNGzakR48eKh0rzAm5XC6WUcploiWXC2JjY5EkSS0WwDxy\n5Ajx8fGUKlWKsmXLMpg4yssAAApGSURBVHbsWN68eUP37t1p27atcr+IiAiCgoJwd3fP0zOo69at\nY//+/fj7+6Onp0dERAT9+/dHU1OTQYMGZcq4evVqVq9eTcuWLenSpQtWVlZ5llP4fomWXC7Q09NT\nixbcvHnz8PPz4/Hjx/j4+GBmZkbnzp05d+4cly5dQltbm4oVKwJgbGyMk5MTGhoamVpWue3FixcE\nBgZy9epVnJ2dMTMzIy4ujpMnT6KlpYWdnR2GhoYAODg4IJfLWbFiBQUKFKBmzZpiPpmQLVHk8qmA\ngAB27NjB3r176dKlC8+ePWPlypVYWFjQrVs3zp8/z5UrV8jIyPhoLCsvz/CVLFkSOzs7Nm7cSGho\nKM2bN8fJyQmFQsH+/ftJT0+nTJkyykJXtWpVChYsSLNmzTAxMcmznML3SxS5fGrjxo3UrFkTJycn\n9u3bh6+vL66urqxatQpLS0vat2+Pt7c3BQsWxNnZOc/zSW/XMkQul5OQkECpUqUICAhQ3s+gTp06\nvHnzhm3btpGSkoKVlZWy0NnZ2an9PV8F9SGKXD6UmprK2rVradasGc+ePcPDw4MlS5bQo0cPbty4\nwfr16zl37hympqasWLFCJXOz3k3WPXDgABMmTKBcuXJIksT58/9t7+5jan7/OI4/q9PhkDqnsmwn\npbkrYkvZQe7WbObLLCNMGsnKWWZzd0IhG+YYa1iSzUhyv6ZMztTcjC1WKwtjZialUg7VHOko/f5o\nzk8/uf1FfY/348/P55zruj7b2WvX57rOdV1FFBUVMXPmTCZMmMDbt2/Jzc3FbDbj7++Pi4vLH2+r\n+HeTiQc7VV9fj1qtJikpCZVKZfs7y8GDB+nTpw/Nzc3ExMSgUCj++G4inzx//pzIyEiSkpKYPn06\nFouFoqIiEhMTGT58OOnp6Tg7O2M0Grl27RpZWVnyiip+mvy92k6p1WqsViuPHz/m9evXtusVFRWM\nHz+euLi4bg04+O9a1JCQEAD69u3LxIkT2bZtG8XFxSQkJGC1WklISODkyZMScOKXyIoHO6ZUKvnn\nn3/YtWsXbm5ulJeX8+bNG9uGkkC3LgbXarXU1tZy6tQpVq5caWtPSEgI/v7+5OXl0dTURFpamozB\niV8mY3J2btiwYXh4eFBaWoqnpydpaWkoFIo/fhD0p9UMDQ0NODo6YrVaUavVWCwWcnNzbcuboH1N\ncGVlJcuXL2f27Nm4ubn16LWoomeTMbm/hNVqta2z7a4dfQsKCsjMzMRsNjNp0iQWLVqESqVi9+7d\nFBcXEx4eTkREBHl5eWRlZXH69GlZCSD+bxJyf5nu2lI7Pz8fg8FAUlIStbW1XLhwgYCAAAwGA05O\nThw/fpyMjAy0Wi2tra3s27fvX7kWVfQ8EnLit3vz5g3x8fEsW7aMadOmUV9fz+TJk9FoNIwZM4Z1\n69ah1WqpqanBYrGgVqtlkkF0GZl4EL9dS0sL9+/fx8/Pj+fPnzN//nz0ej3e3t4YDAaampoYOHAg\ngwYNIjIysrubK+yM9OREl/v0Svzs2TOsVivV1dVYLBamTp2KXq9n5MiRGAwGACIiInB1daW8vJzU\n1NQecXqZsC/SkxNd6lPAmUwm9uzZQ79+/Zg1axYxMTGYzWZqamqIiooCwGKxEBgYSHR0NBqNxrZs\nS4iuJCEnupSDgwNXr15l06ZNpKSk4O/vj5eXFwBVVVX07duXwsJCRo0axY0bN3jy5Amenp49YtcW\nYZ/kdVV0qffv32MwGAgLCyM8PJyGhgZKS0tt518UFxfj6+tLQ0MDjY2NHDlyhMDAwO5utrBj0pMT\nXaqtrY2KigoePnzI0KFD2bt3L05OTrS0tBAcHMyDBw+YOXMmQ4YMwc/Pr1tPthd/Bwk50aVUKhVL\nlixh586dXLx4kcmTJzNnzhx0Oh1ms5mysjKmTp2Kn59fdzdV/CUk5ESXCw8PR6fT0dDQYNt5GMBk\nMlFdXS0TDOKPkjE58Vu9e/eO69ev8+LFCw4dOkRmZiYjRozo7maJv4j05MRvlZOTw7Fjx/Dx8eHE\niRNy7J7446QnJ36r5uZm21bnKpWqu5sj/kISckIIuyY7Awsh7JqEnBDCrknICSHsmoScEMKuScgJ\nIeyahJz4rsrKSgIDA4mKiiIqKoqFCxeydu1aGhsbf7nMc+fOsWHDBgBWr17Ny5cvv/rZkpISKioq\nfrjslpaWTvelO3DgACkpKd/8blhYGOXl5T9c14YNGzh37twPf178eRJy4oe4u7uTmZlJZmam7YCZ\ntLS0Lik7JSXFth1TZ7Kzs38q5IT4nKx4EL9k7NixnDlzBmjv/cyYMYOKigr2799PXl4eJ06coK2t\nDXd3d7Zv345GoyErK4tTp04xYMCADqdwhYWFcfToUQYOHMj27du5f/8+ANHR0SgUCkwmE2VlZWzc\nuBFfX1+2bdtGU1MT7969Y82aNUyYMIGnT5+yfv16VCoVOp3uu+0/efIkOTk5ODs706tXL1JSUnB1\ndQXae5n37t3DbDazefNmdDodVVVVndYrej4JOfHTWltbyc/PJzg42HZt0KBBrF+/nurqag4dOsT5\n8+dRKpVkZGSQnp5OfHw8+/fvx2QyodFo0Ov1uLm5dSg3NzeXV69ecfbsWRobG1m3bh1paWkEBASg\n1+sZP348sbGxLFu2jHHjxlFXV8eCBQu4cuUKqampzJ07l0WLFnHlypXvPkNzczNHjhzBxcWFLVu2\nkJuby+LFiwFQq9VkZGRQWFiI0WgkOzub5OTkTusVPZ+EnPghr1+/tm1b/vHjR0JCQli6dKntflBQ\nEAClpaXU1dURExMDtJ/36u3tTXl5OVqtFo1GA4BOp+PRo0cd6igrK7P1wlxdXTl8+PAX7bhz5w4W\ni4XU1FQAFAoFZrOZx48fExsbC8C4ceO++zxqtZrY2FgcHR158eIF/fv3t90LDQ21PdOTJ0++Wa/o\n+STkxA/5NCb3Nc7OzgAolUpGjx5Nenp6h/v37t3rcN7rx48fvyjDwcGh0+ufUyqVHDhwAHd39w7X\n29racHRsH2JubW39Zhk1NTUYjUYuXbqEh4cHRqPxi3b8b5lfq1f0fDLxILrUqFGjKCsro66uDoDL\nly9TUFCAj48PlZWVNDY20tbWRmFh4RffDQoK4ubNmwC8ffuWiIgIrFYrDg4OfPjwAYDg4GAuX74M\ntPcud+zYAcDgwYO5e/cuQKdlf85sNqPRaPDw8KC+vp5bt25htVpt92/fvg20z+oOHTr0m/WKnk96\ncqJLeXl5kZiYSFxcHCqVit69e2M0GnFzc2PFihVERkai1WrRarW8f/++w3dnzJhBSUkJCxcupLW1\nlejoaJRKJaGhoWzdupVNmzaRmJjIli1buHTpElarFb1eD0B8fDwJCQmYTCaCgoJQKL7+0w4ICMDX\n15d58+bh4+PDqlWrSE5OZsqUKQDU19cTFxdHVVUVW7duBfhqvaLnk11IhBB2TV5XhRB2TUJOCGHX\nJOSEEHZNQk4IYdck5IQQdk1CTghh1yTkhBB27T9M5UrSyEa7BQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd02f11d2b0>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(print_confusion_matrix(cf_matrix, class_names=labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "9XE5c4LJEZXy",
    "outputId": "6f276941-fb3c-468e-d8d1-87b1625413a4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy: 0.828548644338118\n"
     ]
    }
   ],
   "source": [
    "test_accuracy = accuracy_score(y_test_original, y_pred_original)\n",
    "print(\"test accuracy:\", test_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jAISNDCVwvR7"
   },
   "source": [
    "### Performance score for each classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 173
    },
    "colab_type": "code",
    "id": "iFb9Q6A1DcXi",
    "outputId": "3f6ff514-f49d-4d2a-a94d-f94c8912bd28"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1-score</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fear</th>\n",
       "      <td>0.8340</td>\n",
       "      <td>0.8609</td>\n",
       "      <td>0.8087</td>\n",
       "      <td>643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>0.8040</td>\n",
       "      <td>0.7995</td>\n",
       "      <td>0.8086</td>\n",
       "      <td>789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sadness</th>\n",
       "      <td>0.9086</td>\n",
       "      <td>0.9111</td>\n",
       "      <td>0.9061</td>\n",
       "      <td>543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>joy</th>\n",
       "      <td>0.7789</td>\n",
       "      <td>0.7562</td>\n",
       "      <td>0.8030</td>\n",
       "      <td>533</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         f1-score  precision  recall  support\n",
       "fear       0.8340     0.8609  0.8087      643\n",
       "anger      0.8040     0.7995  0.8086      789\n",
       "sadness    0.9086     0.9111  0.9061      543\n",
       "joy        0.7789     0.7562  0.8030      533"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = precision_recall_fscore_support(y_test_original, y_pred_original)\n",
    "score_dict = {\n",
    "  \"precision\": precision.round(4),\n",
    "  \"recall\": recall.round(4),\n",
    "  \"f1-score\": fscore.round(4),\n",
    "  \"support\": support.round(4)\n",
    "}\n",
    "score_df = pd.DataFrame(score_dict, index=labels)\n",
    "score_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cHi1Gw7Oz1zG"
   },
   "source": [
    "### Performance score using micro average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 80
    },
    "colab_type": "code",
    "id": "KP69PDE9D_jz",
    "outputId": "092c1a18-323c-402c-91c3-9bcfef72a24e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1-score</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>score</th>\n",
       "      <td>0.8285</td>\n",
       "      <td>0.8285</td>\n",
       "      <td>0.8285</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       f1-score  precision  recall support\n",
       "score    0.8285     0.8285  0.8285    None"
      ]
     },
     "execution_count": 30,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = precision_recall_fscore_support(y_test_original, y_pred_original, average=\"micro\")\n",
    "score_dict = {\n",
    "  \"precision\": precision.round(4),\n",
    "  \"recall\": recall.round(4),\n",
    "  \"f1-score\": fscore.round(4),\n",
    "  \"support\": support\n",
    "}\n",
    "score_df = pd.DataFrame(score_dict, index=[\"score\"])\n",
    "score_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IWXNCktiz8Vb"
   },
   "source": [
    "### Performance score using macro average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 80
    },
    "colab_type": "code",
    "id": "g4e0vsNIydDs",
    "outputId": "55b1571b-4072-4375-e9bf-464c04876875"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1-score</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>score</th>\n",
       "      <td>0.8314</td>\n",
       "      <td>0.8319</td>\n",
       "      <td>0.8316</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       f1-score  precision  recall support\n",
       "score    0.8314     0.8319  0.8316    None"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = precision_recall_fscore_support(y_test_original, y_pred_original, average=\"macro\")\n",
    "score_dict = {\n",
    "  \"precision\": precision.round(4),\n",
    "  \"recall\": recall.round(4),\n",
    "  \"f1-score\": fscore.round(4),\n",
    "  \"support\": support\n",
    "}\n",
    "score_df = pd.DataFrame(score_dict, index=[\"score\"])\n",
    "score_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wFJFH7u6z-oG"
   },
   "source": [
    "### Performance score using weighted average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 80
    },
    "colab_type": "code",
    "id": "z99JkHjuzv68",
    "outputId": "f2b1675b-1673-482a-a993-04fb5de9afe7"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1-score</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>score</th>\n",
       "      <td>0.829</td>\n",
       "      <td>0.8302</td>\n",
       "      <td>0.8285</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       f1-score  precision  recall support\n",
       "score     0.829     0.8302  0.8285    None"
      ]
     },
     "execution_count": 32,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = precision_recall_fscore_support(y_test_original, y_pred_original, average=\"weighted\")\n",
    "score_dict = {\n",
    "  \"precision\": precision.round(4),\n",
    "  \"recall\": recall.round(4),\n",
    "  \"f1-score\": fscore.round(4),\n",
    "  \"support\": support\n",
    "}\n",
    "score_df = pd.DataFrame(score_dict, index=[\"score\"])\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "Y83dx5SyoBR8",
    "outputId": "0d58dac8-378d-4f59-9471-25c23d188bfc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kappa: 0.7694247720631184\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import cohen_kappa_score\n",
    "\n",
    "kappa_score = cohen_kappa_score(y_test_original, y_pred_original, labels=labels)\n",
    "print(\"kappa:\", kappa_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5womFgI2kjed"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "EmoInt-Emotion-Analysis-Bidirectional-LSTM-Threshold-0.33.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}