BERT_NER.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "BERT --NER ",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNAhUe7otkfQmbTWUfhK8m/",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ludwigwittgenstein2/Research/blob/master/BERT_NER.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rNaxA4ofQIGk"
      },
      "source": [
        "#BERT Fine-Tuning and Prediction on BioMedical Data\n",
        "\n",
        "\n",
        "\n",
        "Steps: \n",
        "\n",
        "1) Load the Data\n",
        "2) Apply BERT \n",
        "3) BERT for Fine-Tuning\n",
        "4) Fit BERT for Name-Entitiy Recognition\n",
        "5) "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PEg8sC0P4bro"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AkDVE7gzQCzr",
        "outputId": "f567fd05-d4f8-4735-aec9-98c8604b9222"
      },
      "source": [
        "#\n",
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ed/d5/f4157a376b8a79489a76ce6cfe147f4f3be1e029b7144fa7b8432e8acb26/transformers-4.4.2-py3-none-any.whl (2.0MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0MB 6.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 38.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 40.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=c1f68902c101b0298383c9cb70a5f3db3fcb31485714319a38f4c80b81a0aee0\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.4.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "smZTpxGQBGO8"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y7IRqZCTB34-"
      },
      "source": [
        "#Load the Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7x-3mO8IiuA_"
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np \n",
        "from tqdm import tqdm, trange"
      ],
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JaS0TTYsRTqG"
      },
      "source": [
        "data = pd.read_csv('/content/sample_data/biomed.csv', encoding=\"latin1\")"
      ],
      "execution_count": 100,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "itUHps1hqziu",
        "outputId": "132b7c0d-d4fe-48a0-adbb-ebd18ee4d83d"
      },
      "source": [
        "data.head()"
      ],
      "execution_count": 101,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Sentence #           Word  POS Tag\n",
              "0  Sentence: 1      Thousands  NNS   O\n",
              "1          NaN             of   IN   O\n",
              "2          NaN  demonstrators  NNS   O\n",
              "3          NaN           have  VBP   O\n",
              "4          NaN        marched  VBN   O"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y4Ca5co8rtPW"
      },
      "source": [
        "class SentenceGetter(object):\n",
        "\n",
        "    def __init__(self, data):\n",
        "        self.n_sent = 1\n",
        "        self.data = data\n",
        "        self.empty = False\n",
        "        agg_func = lambda s: [(w, p, t) for w, p, t in zip(s[\"titles\"].values.tolist(),\n",
        "                                                           s[\"pub_date\"].values.tolist(),\n",
        "                                                           s[\"abstract\"].values.tolist())]\n",
        "        self.grouped = self.data.groupby(\"PMID\").apply(agg_func)\n",
        "        self.sentences = [s for s in self.grouped]\n",
        "\n",
        "    def get_next(self):\n",
        "        try:\n",
        "            s = self.grouped[\"PMID: {}\".format(self.n_sent)]\n",
        "            self.n_sent += 1\n",
        "            return s\n",
        "        except:\n",
        "            return None"
      ],
      "execution_count": 102,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q54eaLsLsSQu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "outputId": "579fdb92-8a4c-4ff6-c0f2-9733a2656614"
      },
      "source": [
        "getter = SentenceGetter(data)"
      ],
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-103-f4b1e012b3bb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mgetter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSentenceGetter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-102-c3efc8eeb88b>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m      8\u001b[0m                                                            \u001b[0ms\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"pub_date\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m                                                            s[\"abstract\"].values.tolist())]\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrouped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"PMID\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magg_func\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msentences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0ms\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrouped\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mgroupby\u001b[0;34m(self, by, axis, level, as_index, sort, group_keys, squeeze, observed, dropna)\u001b[0m\n\u001b[1;32m   6523\u001b[0m             \u001b[0msqueeze\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6524\u001b[0m             \u001b[0mobserved\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobserved\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6525\u001b[0;31m             \u001b[0mdropna\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdropna\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   6526\u001b[0m         )\n\u001b[1;32m   6527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/groupby/groupby.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, obj, keys, axis, level, grouper, exclusions, selection, as_index, sort, group_keys, squeeze, observed, mutated, dropna)\u001b[0m\n\u001b[1;32m    531\u001b[0m                 \u001b[0mobserved\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobserved\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m                 \u001b[0mmutated\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmutated\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 533\u001b[0;31m                 \u001b[0mdropna\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    534\u001b[0m             )\n\u001b[1;32m    535\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/groupby/grouper.py\u001b[0m in \u001b[0;36mget_grouper\u001b[0;34m(obj, key, axis, level, sort, observed, mutated, validate, dropna)\u001b[0m\n\u001b[1;32m    784\u001b[0m                 \u001b[0min_axis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgpr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgpr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    785\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 786\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgpr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    787\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgpr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mGrouper\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mgpr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkey\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m             \u001b[0;31m# Add key to exclusions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 'PMID'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jg8PhWDBsU23",
        "outputId": "53cec2b4-7121-48e5-e3bb-9514ffdb9ea9"
      },
      "source": [
        "sentences = [[word[0] for word in sentence] for sentence in getter.sentences]\n",
        "sentences[0]"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Clinical relevance of antibodies against serotonin and gangliosides in patients with primary fibromyalgia syndrome.']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GIX3yoMl91Y9",
        "outputId": "6b1c85b8-433c-46bf-a07d-f143228c4d0a"
      },
      "source": [
        "labels = [[s[2] for s in sentence] for sentence in getter.sentences]\n",
        "print(labels[0])"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['The fibromyalgia syndrome (FMS) is a non-articular rheumatic disorder associated with disturbances in serotonin metabolism. In order to evaluate whether patients with FMS suffer from an autoimmune disorder, we tested sera from 50 clinically well-defined FMS patients for non-organ-specific and organ-specific antibodies by enzyme-linked immunosorbent assay and immunofluorescence test. Common antibodies against nuclei, mitochondria, and microsomes were not increased in these patients compared to healthy controls. However, 74% had antibodies against serotonin and gangliosides. The clinical and diagnostic relevance of these antibodies is supported by the absence of anti-serotonin antibodies in other rheumatic disorders such as rheumatoid arthritis, polymyalgia rheumatica, and collagen diseases. These antibodies may belong to the group of antireceptor antibodies, considering the fact that gangliosides are an important component of the serotonin receptor. It remains to be determined whether these antibodies are of pathogenetic relevance, interfering with serotonin binding and thereby inducing symptoms associated with FMS.']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VWz-K1DhXTSZ"
      },
      "source": [
        "tag_values = list(set(data[\"pub_date\"].values))\n",
        "tag_values.append(\"titles\")\n",
        "tag2idx = {t: i for i, t in enumerate(tag_values)}"
      ],
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AeOIyXPgYECk"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DzsrybwgYDdy"
      },
      "source": [
        "#Apply BERT"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gJBSnqD_ZfgH",
        "outputId": "965d1b7e-507f-4b84-ce62-cb6979e1968d"
      },
      "source": [
        "!pip install torch\n",
        "!pip install keras\n",
        "!pip install sklearn\n",
        "\n"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.8.0+cu101)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.7.4.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch) (1.19.5)\n",
            "Requirement already satisfied: keras in /usr/local/lib/python3.7/dist-packages (2.4.3)\n",
            "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.7/dist-packages (from keras) (1.4.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras) (3.13)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras) (2.10.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from h5py->keras) (1.15.0)\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.7/dist-packages (0.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from sklearn) (0.22.2.post1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (1.0.1)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (1.19.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "Omln7kAMYGHh",
        "outputId": "bcf62e7e-17a7-408b-9e77-d2728eeedc4c"
      },
      "source": [
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertTokenizer, BertConfig\n",
        "\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "torch.__version__"
      ],
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'1.8.0+cu101'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OITRGEfpZe15"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EL--a0PHYIER"
      },
      "source": [
        "MAX_LEN = 75\n",
        "bs = 32"
      ],
      "execution_count": 80,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tgq4CZ-eafFs"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()"
      ],
      "execution_count": 81,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "YgNLOI3uagky",
        "outputId": "4b8916f0-420e-4aa2-c48a-fa593ab04061"
      },
      "source": [
        "torch.cuda.get_device_name(0)"
      ],
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Tesla K80'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hHMxZIrabAw-"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False)\n"
      ],
      "execution_count": 83,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0DMyH8nXbBam"
      },
      "source": [
        "def tokenize_and_preserve_labels(sentence, text_labels):\n",
        "    tokenized_sentence = []\n",
        "    labels = []\n",
        "\n",
        "    for word, label in zip(sentence, text_labels):\n",
        "\n",
        "        # Tokenize the word and count # of subwords the word is broken into\n",
        "        tokenized_word = tokenizer.tokenize(word)\n",
        "        n_subwords = len(tokenized_word)\n",
        "\n",
        "        # Add the tokenized word to the final tokenized word list\n",
        "        tokenized_sentence.extend(tokenized_word)\n",
        "\n",
        "        # Add the same label to the new list of labels `n_subwords` times\n",
        "        labels.extend([label] * n_subwords)\n",
        "\n",
        "    return tokenized_sentence, labels"
      ],
      "execution_count": 84,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ez-aWlfqbDwE"
      },
      "source": [
        "tokenized_texts_and_labels = [\n",
        "    tokenize_and_preserve_labels(sent, labs)\n",
        "    for sent, labs in zip(sentences, labels)\n",
        "]"
      ],
      "execution_count": 85,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-3kdnuKubFwq"
      },
      "source": [
        "tokenized_texts = [token_label_pair[0] for token_label_pair in tokenized_texts_and_labels]\n",
        "labels = [token_label_pair[1] for token_label_pair in tokenized_texts_and_labels]\n"
      ],
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tuNyfWNbbHfZ"
      },
      "source": [
        "input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
        "                          maxlen=MAX_LEN, dtype=\"long\", value=0.0,\n",
        "                          truncating=\"post\", padding=\"post\")"
      ],
      "execution_count": 93,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mWtoNtWabJap",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "c93abcdd-d856-4fd2-f904-514162945a85"
      },
      "source": [
        "tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels],\n",
        "                     maxlen=MAX_LEN, value=tag2idx[\"PAD\"], padding=\"post\",\n",
        "                     dtype=\"long\", truncating=\"post\")\n"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-94-4546d9544165>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels],\n\u001b[0;32m----> 2\u001b[0;31m                      \u001b[0mmaxlen\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMAX_LEN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtag2idx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"PAD\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpadding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"post\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m                      dtype=\"long\", truncating=\"post\")\n",
            "\u001b[0;31mKeyError\u001b[0m: 'PAD'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vjcAal_9kejT"
      },
      "source": [
        "attention_masks = [[float(i != 0.0) for i in ii] for ii in input_ids]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "iKrpBsxskf8w"
      },
      "source": [
        "tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels],\n",
        "                     maxlen=MAX_LEN, value=tag2idx[\"titles\"], padding=\"post\",\n",
        "                     dtype=\"str\", truncating=\"post\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XYSZl5v7lRs0"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LWoo4kZhknH_"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        },
        "id": "4CUSCiENkh4L",
        "outputId": "645ffc1b-0547-4c33-84bf-4d2427429ae9"
      },
      "source": [
        "tr_inputs = torch.tensor(tr_inputs)\n",
        "val_inputs = torch.tensor(val_inputs)\n",
        "tr_tags = torch.tensor(tr_tags)\n",
        "val_tags = torch.tensor(val_tags)\n",
        "tr_masks = torch.tensor(tr_masks)\n",
        "val_masks = torch.tensor(val_masks)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-36-d312b460b00a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtr_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mval_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtr_tags\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_tags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mval_tags\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_tags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtr_masks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_masks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'tr_inputs' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nH3pAILeXNEh"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UJYguF7Rkjb7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "outputId": "32bcdf57-e2bf-4291-98cf-5d423a988633"
      },
      "source": [
        "#NER\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from tqdm import tqdm, trange\n",
        "\n",
        "data = pd.read_csv(\"ner_dataset.csv\", encoding=\"latin1\").fillna(method=\"ffill\")\n",
        "data.tail(10)"
      ],
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1048565</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>impact</td>\n",
              "      <td>NN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048566</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>.</td>\n",
              "      <td>.</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048567</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>Indian</td>\n",
              "      <td>JJ</td>\n",
              "      <td>B-gpe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048568</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>forces</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048569</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>said</td>\n",
              "      <td>VBD</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048570</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>they</td>\n",
              "      <td>PRP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048571</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>responded</td>\n",
              "      <td>VBD</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048572</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>to</td>\n",
              "      <td>TO</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048573</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>the</td>\n",
              "      <td>DT</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048574</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>attack</td>\n",
              "      <td>NN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              Sentence #       Word  POS    Tag\n",
              "1048565  Sentence: 47958     impact   NN      O\n",
              "1048566  Sentence: 47958          .    .      O\n",
              "1048567  Sentence: 47959     Indian   JJ  B-gpe\n",
              "1048568  Sentence: 47959     forces  NNS      O\n",
              "1048569  Sentence: 47959       said  VBD      O\n",
              "1048570  Sentence: 47959       they  PRP      O\n",
              "1048571  Sentence: 47959  responded  VBD      O\n",
              "1048572  Sentence: 47959         to   TO      O\n",
              "1048573  Sentence: 47959        the   DT      O\n",
              "1048574  Sentence: 47959     attack   NN      O"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NwYMHPrmOnzw"
      },
      "source": [
        "class SentenceGetter(object):\n",
        "\n",
        "    def __init__(self, data):\n",
        "        self.n_sent = 1\n",
        "        self.data = data\n",
        "        self.empty = False\n",
        "        agg_func = lambda s: [(w, p, t) for w, p, t in zip(s[\"Word\"].values.tolist(),\n",
        "                                                           s[\"POS\"].values.tolist(),\n",
        "                                                           s[\"Tag\"].values.tolist())]\n",
        "        self.grouped = self.data.groupby(\"Sentence #\").apply(agg_func)\n",
        "        self.sentences = [s for s in self.grouped]\n",
        "\n",
        "    def get_next(self):\n",
        "        try:\n",
        "            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n",
        "            self.n_sent += 1\n",
        "            return s\n",
        "        except:\n",
        "            return None"
      ],
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iGcn0MZxOI4p"
      },
      "source": [
        "getter = SentenceGetter(data)"
      ],
      "execution_count": 109,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RvMNYDBnOsc1",
        "outputId": "531a77ab-4375-401e-d669-0b026d546a66"
      },
      "source": [
        "sentences = [[word[0] for word in sentence] for sentence in getter.sentences]\n",
        "sentences[0]"
      ],
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Thousands',\n",
              " 'of',\n",
              " 'demonstrators',\n",
              " 'have',\n",
              " 'marched',\n",
              " 'through',\n",
              " 'London',\n",
              " 'to',\n",
              " 'protest',\n",
              " 'the',\n",
              " 'war',\n",
              " 'in',\n",
              " 'Iraq',\n",
              " 'and',\n",
              " 'demand',\n",
              " 'the',\n",
              " 'withdrawal',\n",
              " 'of',\n",
              " 'British',\n",
              " 'troops',\n",
              " 'from',\n",
              " 'that',\n",
              " 'country',\n",
              " '.']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 110
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "10UG6J1JOvmB",
        "outputId": "e89bac29-fc77-4356-ff47-88f65fa4d0ee"
      },
      "source": [
        "labels = [[s[2] for s in sentence] for sentence in getter.sentences]\n",
        "print(labels[0])"
      ],
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['O', 'O', 'O', 'O', 'O', 'O', 'B-geo', 'O', 'O', 'O', 'O', 'O', 'B-geo', 'O', 'O', 'O', 'O', 'O', 'B-gpe', 'O', 'O', 'O', 'O', 'O']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fPBqyRcvOxpv"
      },
      "source": [
        "tag_values = list(set(data[\"Tag\"].values))\n",
        "tag_values.append(\"PAD\")\n",
        "tag2idx = {t: i for i, t in enumerate(tag_values)}"
      ],
      "execution_count": 112,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "C2mr60K2OzSB",
        "outputId": "e029c8e1-94d2-4244-fede-6aa87dc7546f"
      },
      "source": [
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertTokenizer, BertConfig\n",
        "\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "torch.__version__"
      ],
      "execution_count": 113,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'1.8.0+cu101'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_A-y1HKLO1kv"
      },
      "source": [
        "MAX_LEN = 75\n",
        "bs = 32\n"
      ],
      "execution_count": 114,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d0BwZDI-PZGo"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n"
      ],
      "execution_count": 115,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "-VsKrXnxPawA",
        "outputId": "1d6e2f79-173e-4f9d-d3d5-00a6627e1aac"
      },
      "source": [
        "torch.cuda.get_device_name(0)\n"
      ],
      "execution_count": 116,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Tesla K80'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p74HtK-2PcP2"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False)\n"
      ],
      "execution_count": 117,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fHoy8C_WPdxF"
      },
      "source": [
        "def tokenize_and_preserve_labels(sentence, text_labels):\n",
        "    tokenized_sentence = []\n",
        "    labels = []\n",
        "\n",
        "    for word, label in zip(sentence, text_labels):\n",
        "\n",
        "        # Tokenize the word and count # of subwords the word is broken into\n",
        "        tokenized_word = tokenizer.tokenize(word)\n",
        "        n_subwords = len(tokenized_word)\n",
        "        \n",
        "\n",
        "        # Add the tokenized word to the final tokenized word list\n",
        "        tokenized_sentence.extend(tokenized_word)\n",
        "\n",
        "        # Add the same label to the new list of labels `n_subwords` times\n",
        "        labels.extend([label] * n_subwords)\n",
        "\n",
        "    return tokenized_sentence, labels\n"
      ],
      "execution_count": 118,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "89tvP-kmPgsD"
      },
      "source": [
        "tokenized_texts_and_labels = [\n",
        "    tokenize_and_preserve_labels(sent, labs)\n",
        "    for sent, labs in zip(sentences, labels)\n",
        "]\n"
      ],
      "execution_count": 119,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RWnMqaz8Pj2L"
      },
      "source": [
        "tokenized_texts = [token_label_pair[0] for token_label_pair in tokenized_texts_and_labels]\n",
        "labels = [token_label_pair[1] for token_label_pair in tokenized_texts_and_labels]\n"
      ],
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_SDici2YQIZI"
      },
      "source": [
        "input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
        "                          maxlen=MAX_LEN, dtype=\"long\", value=0.0,\n",
        "                          truncating=\"post\", padding=\"post\")\n"
      ],
      "execution_count": 121,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s1j26rhgQyp6"
      },
      "source": [
        "tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels],\n",
        "                     maxlen=MAX_LEN, value=tag2idx[\"PAD\"], padding=\"post\",\n",
        "                     dtype=\"long\", truncating=\"post\")\n"
      ],
      "execution_count": 122,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8KziLGBVQvOA"
      },
      "source": [
        "attention_masks = [[float(i != 0.0) for i in ii] for ii in input_ids]\n"
      ],
      "execution_count": 123,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gd1Q-PXiQ4RU"
      },
      "source": [
        "tr_inputs, val_inputs, tr_tags, val_tags = train_test_split(input_ids, tags,\n",
        "                                                            random_state=2018, test_size=0.1)\n",
        "tr_masks, val_masks, _, _ = train_test_split(attention_masks, input_ids,\n",
        "                                             random_state=2018, test_size=0.1)\n"
      ],
      "execution_count": 124,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U6SgJ-5YQ6wP"
      },
      "source": [
        "tr_inputs = torch.tensor(tr_inputs)\n",
        "val_inputs = torch.tensor(val_inputs)\n",
        "tr_tags = torch.tensor(tr_tags)\n",
        "val_tags = torch.tensor(val_tags)\n",
        "tr_masks = torch.tensor(tr_masks)\n",
        "val_masks = torch.tensor(val_masks)\n"
      ],
      "execution_count": 125,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WQSOZFuqQ-p1"
      },
      "source": [
        "train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=bs)\n",
        "\n",
        "valid_data = TensorDataset(val_inputs, val_masks, val_tags)\n",
        "valid_sampler = SequentialSampler(valid_data)\n",
        "valid_dataloader = DataLoader(valid_data, sampler=valid_sampler, batch_size=bs)\n"
      ],
      "execution_count": 126,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YbMHV9RyRBjR"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "GS--X6RDREE9",
        "outputId": "c6686b71-aeb3-4b96-c7d7-f460c908e401"
      },
      "source": [
        "import transformers\n",
        "from transformers import BertForTokenClassification, AdamW\n",
        "\n",
        "transformers.__version__\n"
      ],
      "execution_count": 127,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'4.4.2'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 127
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SgR_yyyyREr3",
        "outputId": "335f3a75-b414-4a44-feea-e84e65916b60"
      },
      "source": [
        "model = BertForTokenClassification.from_pretrained(\n",
        "    \"bert-base-cased\",\n",
        "    num_labels=len(tag2idx),\n",
        "    output_attentions = False,\n",
        "    output_hidden_states = False\n",
        ")\n"
      ],
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i9vV41BhRGXz"
      },
      "source": [
        "model.cuda();\n"
      ],
      "execution_count": 129,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eN_HWBDFRJv5"
      },
      "source": [
        "FULL_FINETUNING = True\n",
        "if FULL_FINETUNING:\n",
        "    param_optimizer = list(model.named_parameters())\n",
        "    no_decay = ['bias', 'gamma', 'beta']\n",
        "    optimizer_grouped_parameters = [\n",
        "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "         'weight_decay_rate': 0.01},\n",
        "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "         'weight_decay_rate': 0.0}\n",
        "    ]\n",
        "else:\n",
        "    param_optimizer = list(model.classifier.named_parameters())\n",
        "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
        "\n",
        "optimizer = AdamW(\n",
        "    optimizer_grouped_parameters,\n",
        "    lr=3e-5,\n",
        "    eps=1e-8\n",
        ")\n"
      ],
      "execution_count": 130,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BcnCsQkuRKO6"
      },
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "epochs = 3\n",
        "max_grad_norm = 1.0\n",
        "\n",
        "# Total number of training steps is number of batches * number of epochs.\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(\n",
        "    optimizer,\n",
        "    num_warmup_steps=0,\n",
        "    num_training_steps=total_steps\n",
        ")\n"
      ],
      "execution_count": 131,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LMLMv_omRQhQ",
        "outputId": "63e8e4a3-3506-4537-a2df-3b47018d88fd"
      },
      "source": [
        "!pip install seqeval\n"
      ],
      "execution_count": 132,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: seqeval in /usr/local/lib/python3.7/dist-packages (1.2.2)\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from seqeval) (1.19.5)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from seqeval) (0.22.2.post1)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.0.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IcQir1GoRV2i"
      },
      "source": [
        "\n",
        "from seqeval.metrics import f1_score, accuracy_score\n"
      ],
      "execution_count": 133,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RpzWz0_ARS1T",
        "outputId": "f50a06f5-7a42-475d-d3ee-05c5cf2595d6"
      },
      "source": [
        "## Store the average loss after each epoch so we can plot them.\n",
        "loss_values, validation_loss_values = [], []\n",
        "\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    # Put the model into training mode.\n",
        "    model.train()\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_loss = 0\n",
        "\n",
        "    # Training loop\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        # add batch to gpu\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "        # Always clear any previously calculated gradients before performing a backward pass.\n",
        "        model.zero_grad()\n",
        "        # forward pass\n",
        "        # This will return the loss (rather than the model output)\n",
        "        # because we have provided the `labels`.\n",
        "        outputs = model(b_input_ids, token_type_ids=None,\n",
        "                        attention_mask=b_input_mask, labels=b_labels)\n",
        "        # get the loss\n",
        "        loss = outputs[0]\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "        # track train loss\n",
        "        total_loss += loss.item()\n",
        "        # Clip the norm of the gradient\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
        "        # update parameters\n",
        "        optimizer.step()\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over the training data.\n",
        "    avg_train_loss = total_loss / len(train_dataloader)\n",
        "    print(\"Average train loss: {}\".format(avg_train_loss))\n",
        "\n",
        "    # Store the loss value for plotting the learning curve.\n",
        "    loss_values.append(avg_train_loss)\n",
        "\n"
      ],
      "execution_count": 135,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "Epoch:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "Epoch:  33%|███▎      | 1/3 [18:30<37:00, 1110.08s/it]\u001b[A\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Average train loss: 0.10845937778285426\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "Epoch:  67%|██████▋   | 2/3 [37:00<18:30, 1110.17s/it]\u001b[A\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Average train loss: 0.08093532306550166\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "Epoch: 100%|██████████| 3/3 [55:29<00:00, 1109.80s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Average train loss: 0.07002610940930798\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ATra189RaVJ"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "9Ak6WOvvRdk8",
        "outputId": "5f513475-39e1-4b2b-fb5f-82baf0b453f5"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(loss_values, 'b-o', label=\"training loss\")\n",
        "plt.plot(validation_loss_values, 'r-o', label=\"validation loss\")\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Learning curve\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "\n",
        "plt.show()\n"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvoAAAGaCAYAAAB+A+cSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVhVVdvH8e85zIozKM7iBE6oOCumoiUaZs5hioqZjU85lFrm0FNZqVlPswaaqGkWRuaU5JA5kZriAM4jiuAsDszvH76cREA5iB7A3+e6upR11trr3jeny/vss/bahrS0tDRERERERKRQMVo6ABERERERyXsq9EVERERECiEV+iIiIiIihZAKfRERERGRQkiFvoiIiIhIIaRCX0RERESkEFKhLyIiecrb25uBAwdaOgwRkUeeCn0RkXxg69atuLm5ERgYaOlQRESkkLC2dAAiIlK4rFy50tIhiIgIuqIvIiLZSEpKIiEhwexxtra22NraPoCI8qf4+HhLhyAikiUV+iIiBcyxY8d444038PLyon79+nh7e/PRRx9x/fr1DP0OHz7MpEmTePLJJ2ncuDENGzakZ8+eLF68ONMxP//8c9zc3Dh48CBTpkzhsccew8PDg507dxISEoKbmxubN28mMDCQTp06Ub9+fTp37sySJUsyHSurNfrpbYcPH+b555+ncePGNGnShP/85z/ExcVlOkZUVBQBAQE0atSIFi1aMGbMGC5cuICbmxtjx47NUZ4SExOZNWsW3bt3p2HDhjRp0oSePXsyb948U5+xY8fi5uaW5fg75zp16hRubm58/vnnLF++nJ49e+Lh4cF7773H1KlTcXNzIyoqKtNxrl69ioeHBy+99FKG9k2bNhEQEEDTpk1p0KAB3bp144cffsjRuYmI5ISW7oiIFCB79uxh0KBBFC9enH79+lGuXDmioqIIDg7mn3/+ITg4GBsbGwDCw8PZtm0b7du3p1KlSty4cYOVK1cyfvx4Lly4wPDhwzMdf/To0djb2xMQEACAs7Mz0dHRAMyYMYObN2/Sr18/bG1t+eGHHxg7dixVqlShSZMm94z97Nmz+Pv706lTJ958802ioqJYtGgR8fHxBAUFmfodO3aMZ599ltTUVAYOHEi5cuVYv349zz33XI7zlJiYyNChQwkPD8fLy4unnnoKOzs7Dhw4wO+//86AAQNyfKw7hYWFERwcjJ+fH8888wyOjo7Url2b7777jtDQUNzd3TP0X7FiBQkJCfTo0cPUtmjRIiZOnEijRo144YUXcHBwYNOmTUyaNIkTJ04wZsyYXMcnIpJOhb6ISAHy1ltv4ezszE8//YSjo6OpvVWrVrzyyissXbqUnj17AtC9e3f8/PwyjB88eDCDBg1i5syZBAQEmD4UpCtevDizZ8/G2vrffx527twJ3Cqef/rpJ9OyHB8fHzp27Mj8+fNzVOgfP36cGTNm0LVrV1Ob0WhkwYIFHDlyhOrVqwO3PlDEx8ezYMEC03EHDBjA66+/zt69e3OUp++//57w8HCGDx/OyJEjM7yWmpqao2Nk59ChQ/z666/UqFEjQ3v9+vVZunQpo0ePxsrKytT+yy+/ULJkSdq1awdAbGws7733Hk8++STTp0839Xv22Wd57733mDNnDv3796dy5cr3FaeIiJbuiIgUEPv372f//v34+vqSmJjIhQsXTP81adKEIkWKsHHjRlP/IkWKmP6ekJDAxYsXuXTpEm3atCE+Pp4jR45kmmPQoEEZivzb9e/fP8Pa+3LlyuHq6sqxY8dyFH/ZsmUzFPkALVu2BG59CABISUnhzz//xMPDI9OHh/RvGXJi6dKllChRgpdffjnTa0bj/f3T165du0xFPkCPHj2Ii4vL8Ds4efIkO3bswNfX15S7VatWkZiYSO/evTP8Di9cuIC3tzepqals2rTpvmIUEQFd0RcRKTAOHz4M3FpP//nnn2fZ59y5c6a/X7t2jS+++IIVK1Zw5syZTH2vXLmSqa1atWrZzp/VFeaSJUualvbcS3bjAS5dugTAhQsXuH79Oq6urpn6ZtWWnePHj1OnTh3s7OxyPCanssvRk08+yYcffkhoaCiPPfYYAKGhoaSlpdG9e3dTv/Tf4+DBg7Od4/bfo4hIbqnQFxEpYAICAmjbtm2WrxUvXtz091GjRrFu3Tr69u1Ls2bNKFmyJFZWVqxfv545c+ZkuYTF3t4+23nv90r47ctZ7pSWlnZfx84tg8GQZXtycnK2YxwcHLJsL1WqFO3atSMsLIz4+HgcHR0JDQ2lRo0aeHh4mPqln+tHH31E2bJlszyWlu2ISF5QoS8iUkBUrVoVuFVwt27d+q59r1y5wrp16+jevTvvvvtuhtfy87KQ0qVLU6RIEY4ePZrptazaslOtWjWOHDlCYmLiXbf6LFGiBHDrG4X0bxfg1pKb3OjRowdhYWGsXLkSV1dXTpw4wahRozLFBrc+GNzr9ygicj+0Rl9EpICoW7cutWvXZuHChVkWosnJyaYlMOlX3++8Uh4bG5vl9pr5hZWVFW3btiUiIoLt27dneO32nXnupVu3bly+fJmvvvoq02u35yS96L7zw8/s2bPNiPpf7dq1o1SpUoSGhhIaGorRaMywbAegS5cu2Nra8vnnn3Pz5s1Mx7h69SqJiYm5ml9E5Ha6oi8iko9s3rw5y4dUlSpVCj8/Pz7++GMGDRrEU089Ra9evahZsyY3b97k+PHjrF69mpEjR9KzZ08cHR1p06YNv/76K/b29jRo0IDo6GgWLVpEpUqVTB8I8qPXX3+dv/76i+eee44BAwbg4uLCunXruHDhApD9cpvb+fv7s3btWr7++mt2796Nl5cXtra2HDp0iKNHjzJnzhwAfH19mTFjBhMmTODIkSOULFmSDRs2cPHixVzFbmNjg6+vL/PmzWPPnj20bt2acuXKZejj4uLCpEmTGD9+PF27duWpp56iYsWKXLhwgQMHDhAWFsayZcuoVKlSrmIQEUmnQl9EJB/ZsGEDGzZsyNTu6uqKn58fderUYcmSJXz77besWbOGhQsXUrRoUSpWrEiPHj1o1aqVaczUqVOZPn06a9asYcmSJVSrVo0RI0ZgbW3NuHHjHuZpmaV69erMnz+fjz76iLlz52JnZ0f79u2ZMGECnTp1ytENtra2tgQFBREUFMRvv/3GJ598gp2dHVWrVjVtPwrg6OjIzJkzmTJlCt9++y1FihThiSeeYOrUqTRr1ixX8T/99NMEBwdz/fr1TFfz0/Xq1Ytq1aoRFBTEokWLuHr1KiVLlsTV1ZXXXnsNZ2fnXM0tInI7Q5ql7oASERExw549e+jVqxejRo3i+eeft3Q4IiL5ntboi4hIvnPn2vW0tDS+++47AN3AKiKSQ1q6IyIi+U737t1p2bIltWvX5saNG6xdu5Zt27bRtWtX6tevb+nwREQKBIsv3UlMTOSzzz4jNDSUK1eu4O7uzogRIzKsM81KREQEISEhREREcODAAZKSkti/f3+mfteuXSMwMJBdu3axe/duLl++zJQpUzKs0bzdjh07mDp1Kvv27cPR0ZEuXbowatSobPdNFhGRvPfxxx+zdu1aYmJiSE5OplKlSnTr1o1hw4ZhY2Nj6fBERAoEixf6I0eO5Pfff8ff35+qVauyZMkS9uzZQ3BwMI0bN8523Oeff84333yDm5sbN27c4MiRI1kW+qdOnaJjx46UL1+eKlWqsHXr1mwL/cjISPr160fNmjXp06cPMTExBAUF0aZNG7755ps8PW8RERERkQfJokt3IiIiWLZsGePGjTM9Cvzpp5/G19eXadOmMX/+/GzH+vn5MWzYMOzt7Xn//fc5cuRIlv3Kli3Lhg0bKFu2LJGRkTz99NPZHvOTTz6hZMmSBAcHU7RoUQAqVarE+PHj2bx58z2/ZRARERERyS8sejPuypUrsbGxoU+fPqY2Ozs7evfuzfbt24mNjc12rJOT010f1Z7O1tY220eM3y4+Pp5Nmzbx9NNPm4p8uLVOtEiRIqxYseKexxARERERyS8sekU/MjISV1fXDIU1gIeHB2lpaURGRuaoSM8L+/fvJzk5OdNNXra2ttSpU4fIyEizj3nx4jVSUx/uyqgyZRw5fz7+oc5ZkClf5lPOzKN8mUf5Mo/yZR7lyzzKl3kslS+j0UCpUkWzfM2ihX5cXFymJwYCpgeF3O2K/oOI5fa574xn586dZh8zNTXtoRf66fNKzilf5lPOzKN8mUf5Mo/yZR7lyzzKl3nyW74sWujfvHkzy90T0p96mNVj4B9kLHDrCn5W8dy5p3NOlCnjeN9x5YazczGLzFtQKV/mU87Mo3yZR/kyj/JlHuXLPMqXefJbvixa6Nvb25OUlJSpPb3Az8ljzvMyFri13WdW8eTkfoA7nT8f/9A/2Tk7FyMu7upDnbMgU77Mp5yZR/kyj/JlHuXLPMqXeZQv81gqX0ajIduLyxa9GdfZ2TnL5Tnpy2ge1vr89Fhun/vOeB5mLCIiIiIi98uihb67uztHjx7l2rVrGdp37dplev1hqV27NtbW1uzZsydDe2JiIpGRkdSpU+ehxSIiIiIicr8sWuj7+PiQlJTE4sWLTW2JiYmEhITg6elpulH39OnTHD58+IHGUqxYMVq1akVoaGiGDx6hoaFcv34dHx+fBzq/iIiIiEhesuga/YYNG+Lj48O0adOIi4ujSpUqLFmyhNOnTzNlyhRTvzFjxhAeHp7hybfR0dGEhoYCsHv3bgC++uor4NY3Ad7e3qa+8+bN48qVK5w7dw7A9Fh1gJdeesnUb8SIETzzzDMMHDjQ9GTc2bNn89hjj9G6desHlAURERF5kG7cuEZ8/GViY1NITU21dDgFRmysUfkyQ17ny8rKBkfHEjg4ZL11Zk4Y0tLSLLoPUEJCAp9++ilLly7l8uXLuLm5MXLkyAyF9cCBAzMV+lu3bsXf3z/LY/bo0YMPP/zQ9LO3tzfR0dFZ9r39mADbtm1j2rRp7Nu3D0dHR7p27crIkSMpUqSI2eemm3HzP+XLfMqZeZQv8yhf5lG+7i0pKZGLF2MpWdIJBwcHUlLy1/aH+Zm1tZHkZBX6OZWX+UpLSyMpKYFLl85RqlRZbGwy7wqZ7m4341q80C/MVOjnf8qX+ZQz8yhf5lG+zKN83duFC7HY2ztQpEgxFa5mUr7M8yDyde3aVRITb1CqVPabwuTbXXdEREREHqTk5ETs7BwsHYZIrtjbO5CUlHnr95yy6Bp9yTub98YQsv4wF64kULq4HT3b1aBVPRdLhyUiImJRqakpGI1Wlg5DJFeMRitSU1NyPV6FfiGweW8M36+IIvH/vy46fyWB71dEAajYFxGRR57BYLB0CCK5cr/vXS3dKQRC1h82FfnpEpNTCVn/YLckFREREZH8S4V+IXD+SoJZ7SIiIiJ34+XVlFmzvsn12MDAb/M4onvbsWMbXl5N2bFj20OfO79SoV8IlClul2W7tZWB2Es3HnI0IiIi8qDt2bObwMBvuXpVuy5J9lToFwI929XA1jrjr9LayoABmBgUzoZdp9EuqiIiIoXHvn27mT17FvHxD6bQ/+OPjQwZ8lyuxw4aNDSPI5Lc0M24hUD6Dbd37rpTu1JJApftY/aKKHYeOsegLu4UL5L9AxdERESk8ElJSSElJQVb25zXAHZ2drneF97OLuuVBvLwqdAvJFrVc6FVPZdMD08Z7deY38NPEvLnYSYEhjOkizsNazpZMFIRERG5H4GB3zJ79iwA+vR5ytS+ePGvlC9fAS+vpvTp40ft2m4EB88mOvoUM2Z8iadnUxYsCObPP9dy4sRxbt68SbVqrgwcOJgOHTplmMPLqylDhz7PkCHPZ5jzxx9DCQz8lr/+Wg9Au3bejBw5Bnt7+wxjhwwZxtChw80em5Bwk6+//pzVq1eSmJiEp2cTRo8eR48eXTMc0xx//PE78+bN4fjxYxQpUpQ2bdry4ov/oWTJkqY+J0+e4JtvPmf37gji469SokRJPDwa8sYbb+PoeOthVGFhq1iwIJiTJ09gMBhwcXHB1/dp+vb1Mzumh0WFfiFnNBjwaVGF+q6lmbl0L5/9FEH7xhXp16EmdrbaV1hERMRc6c+uOX8lgTIWeHZNu3benD59ilWrVvCf/4ykRIlbBWvJkqVMff7+ewtr1vxOjx59KFasGE5Oty7y/fTTQtq0eYzHH/chOTmJsLDfeeedsXz88ae0bu11z7nHj3+TChUq8cILr3LgQBRLl/5CyZKleOml/+TJ2Pffn8yaNavp0sWXOnXqsXPnDt5443VzU2SyfPlSPvhgMvXqNeDFF/9DbOxZfv55EZGRe5k1ay52dnYkJSUxcuSrWFkZ6devPyVKlODs2bNs2vQX8fFXcXR05O+/tzBp0tu0a9eBp57qQUpKCseOHWX37l0q9MXyKpV15J1BzVjy5xFWhZ8g8tgFnutWlxoVSlg6NBERkQIjPzy7pmbNWri51WHVqhW0bdue8uUrZOpz8uQJgoN/pEqVqhnaf/jhZ+zs/r2C3qtXPwICnmXRovk5KvTd3evy5ptvm36+fPkyy5aF5qjQv9fY/fujWLNmNX5+A3n55dcA6NmzDx98MJlDhw7c8/h3Sk5O5uuvP6dmzdp8/vm3pqVLbm7uTJr0NkuXLqF372c4duwIZ85EM2vW99SpU880/vZvDzZt2oira3Xef3+q2XFYkgr9R4iNtZG+3jXxqFGG75btY0rwDnxbV8W3dTWsrXRftoiIPDo27j7DXxFnzB53+PRlklMybnCRmJzK7OWR/LnztNnH8/IoT5sG5c0edy+enk0zFflAhiL/ypUrpKam4uHRmLCwVTk67tNP98rwc8OGjfjzz7VcuxZP0aKO9zV269ZNAPTo0TtDv169+rF8+dIcxXe7qKh9XLx4gWHDXsxwf4K39+N8+eVnbNq0kd69nzHFvXHjBmrWrI2NjU2mYzk6OhIbe5a9e/dQr159s2OxFBX6jyD3qqV4N6A581cf4NeNx9h95ALDutXFpXQRS4cmIiKSr91Z5N+r3VKyusoPt4rZ778P5NChAyQmJprac/oE1nLlMn5rUaxYcQCuXr16z0L/XmNjYs5gZWWFi0vGDz6VKlXKUWx3iom59UHuzg88RqORSpUqc/bsrdcrVKhIv37PMmfOdyxatIDGjT1p3botTzzhQ5EiRYFb3yysXRvG8OGDKV++Is2aNadDh040a9YiV7E9LCr0H1FF7G0Y1q0eDWs6EbxqP5Nmh9PPuxbtG1XQo8JFRKTQa9Mgd1fS3/hqY5YPpCxT3I4xz3rmRWh54vYr9+l27fqHsWNH0rBhY0aOHEOZMk5YW1uzfPlSVq9emaPjGo1Z39+Xk22872fsg/bqqyN48slubNiwnvDwLXzyyUfMnRvEt9/Oxtm5LKVKlWb27AWEh29hy5ZNbNmyiV9/XcKTTz7FuHETLB1+trRe4xHXvE453h3agloVSxC8aj+f/RTB5Xg9UVdERCQrWT27xtbaSM92NR5yJOZflFu3bg22trZ88skX+Pp2p1WrNvnqirSLS3lSUlJMV+LTnTp1KtfHAzhx4niG9rS0NE6dOkm5chk/6FWvXpNBg4by5Zez+PrrIGJjz/LLLz+bXrexsaFNm7aMGjWGH3/8hZ49+7Bs2a9ER+cuvodBhb5QqpgdI/o1on+nWkQev8g7geHsOBBn6bBERETynVb1XBjUxd30VPoyxe0Y1MX9oe66A+Dg4ABg1gOzjEYjBoOB1NR/98Y/c+Y0Gzasy+vwcqV581YALFnyU4b2n39elKvjubvXpVSp0vzyy08kJSWZ2teu/YO4uFhat24DwLVr8SQnJ2cYW716DaysrEzLmy5fvpThdYPBQI0atQBISMi/F0i1dEeAW9twdmpamTrVSjNr6V6+CNmNV4Py+HWqhYOd3iYiIiLp0p9dY0lubu4AzJz5FR07PoG1tTVt2jxm+gCQldatvVi0aD6jRr3K44935uLFi4SELKZixcocPnzwYYWeLXf3OrRv780PPwRz6dJF0/aaJ0/euiJv7tJia2trXnzxVT74YDKvvjqcTp2eIDb2LD/9tIjq1WvQrVsPALZv38aMGR/Tvn1HqlSpSmpqCqtWrcBgMNCunTcAH374HlevXsHTsylly5bl7Nlbx6lVqzbVqrnmbSLykCo4yaCiU1HG+zcl9K+jLN9ynKgTF3nOty61K5e892ARERF5KGrXdmf48JcJCVnM1q2bSU1NZfHiX+9a6Ddp0oyxY99h3rzv+d//PqF8+Qq8+OKrnDlzOl8U+gDjx79L6dJlCAv7nXXr1tC0aXMmT55C//69zHqyb7quXbtha2vL/Pnf8+WXn1G0aFEef9yHF1541fQE35o1a9G8eUs2bdpAaGgI9vb21KxZi2nT/kf9+g0A6Ny5C7/+uoQlS34iPv4qpUuXwdu7EwEBz2M05t8FMoa0/HAHRCF1/nw8qakPN713Phn3fhw8dYlZS/dx/spNurasSncv10K3DWde5utRoZyZR/kyj/JlHuXr3mJijuPicmvXFWtrI8nJqfcYIenyS74OHtzPkCHPMmHCf3niiS6WDidbDypft7+Hs2I0GihTJusdjwpX1SZ5qlalkkwOaE6bBuVZtvk4783dRvS5a5YOS0RERAqphISbmdp+/PEHjEYjDRs2tkBEBZuW7shdOdhZE9C1Do1qOjFnRRTvzvmb3u1r0LFJJYzahlNERETyUHDwHA4dOoCnZ1MMBiNbt97ayvKpp3pk2odf7k2FvuSIZ21nalQozuwVUfwQdpCIQ+cIeLIupYrZWTo0ERERKSTq1/dg+/ZwZs/+jhs3rlOunAtDhw5n4MAhlg6tQFKhLzlWwtGO13p7sH7naRauOciEwK0M7OxG8zrlLB2aiIiIFAItW7amZcvWlg6j0NAafTGLwWCgfeOKTBrSnLKlivBN6F5mLd3L9ZtJ9x4sIiIiIg+NCn3JFZfSRXhroCfdvVzZui+WCUHhRB2/aOmwREREROT/WbTQT0xMZOrUqXh5eeHh4UHfvn3ZvHnzPcdFREQwadIkevbsSf369XFzc8u2b2pqKrNmzcLb25sGDRrQrVs3li9fnqnf2LFjcXNzy/Rf37597+scCzMro5HuXq68NbAJNlZGpv7wD4vWHCQpH2zFJSIiIvKos+ga/bFjx/L777/j7+9P1apVWbJkCcOGDSM4OJjGjbPfQmn9+vUsXrwYNzc3KleuzJEjR7LtO2PGDGbOnEm/fv2oX78+f/zxByNGjMBoNOLj45Ohr4ODA5MnT87QVrp06fs7yUdA9QrFmTSkOT+uPcSq8JPsPXqBYd3qUbls1nu6ioiIiMiDZ7EHZkVERNCnTx/GjRvH4MGDAUhISMDX15eyZcsyf/78bMeeO3cOR0dH7O3tef/995k7dy779+/P1O/s2bN07NgRPz8/3n77bQDS0tIYMGAAZ86cISwszPQ0s7FjxxIWFsa2bdvy7BwL+gOzciPi8DmClkdx/WYSPR+rwRPNK+frbTgtna+CSDkzj/JlHuXLPMrXvemBWbmnfJlHD8y6zcqVK7GxsaFPnz6mNjs7O3r37s327duJjY3NdqyTkxP29vb3nCMsLIykpCT69+9vajMYDPj5+REdHU1ERESmMSkpKcTHx5t5NpLOo4YT7w5tToPqZfhx7SGm/fAP5y7fsHRYIiIiIo8cixX6kZGRuLq6UrRo0QztHh4epKWlERkZmSdzODo64urqmmkOgH379mVov3btGk2aNKFJkya0aNGCKVOmkJCQcN9xPGqKF7HllZ4NGNLVnaMxV5kYFM7mPTFY6MsjERERkUeSxQr9uLg4ypYtm6nd2dkZ4K5X9M2Zw8nJKUdzODs789xzz/HBBx8wffp0vLy8mDNnDi+//PJ9x/EoMhgMtPWowOSA5lR0dmTWb/v4OnQv8Te0DaeIiEh+8/77k+jdu5vp5zNnTtOypSfLly81e2xeCAz8Fi+vpnl6zJzYsWMbXl5N2bEj75ZyW5LFbsa9efMmNjY2mdrt7G49aTUvrqTfvHkTW1vbHM0xatSoDH18fX0pV64cgYGBbNy4kTZt2pg9f3brpR40Z+diFpk3K87OxZj2mjMhaw8yf2UUR05f4fVnGtPYLfOHPEvJT/kqKJQz8yhf5lG+zKN83V1srBFr63+va97+d/mX4f/vp0vPj5XVrT+NRsM9c3bnWHPMnTubqlWr0a5dhwztRmPuj3k/0s/bysqYq7kfRLxGozHX/59brNC3t7cnKSnz1d304ju9GL/fORITE3M9R0BAAIGBgWzevDlXhf6jeDNudtp7lMe1rCMzl+5lwszNdGxSiT7ta2BrY2XRuPJrvvIz5cw8ypd5lC/zKF/3lpqaarpBUjeXZi99eW16fpydy7F+/Wbg3jm7c6w55s6dTdu27WnTpl2G9oEDA+jff9BD/32lpKSa/jR37gf1/kpNTb3r/+f58mZcZ2fnLJfnxMXFAWS5rCc3c5w7dy7Xczg5OWFjY8Ply5fvOxaBqi7FmDi4GZ2aVOKP7aeYPOdvjsfoHygREZH8xmAwYGdnh5WVZS7IWVtb58lF30edxQp9d3d3jh49yrVr1zK079q1y/T6/apTpw7x8fEcPXo0yznq1Klz1/ExMTEkJSVpL/08ZGtjRf/HazOqXyNuJCTz3txt/Lbp2EP/5kNERKSgWrMmDC+vpkRE7Mz02rx5c2jbthlnz8YAsGvXP4wfP4aePZ+kQ4dW9Oz5JP/733QSEm7edY7s1uj/+ec6Bg7si7d3awYO7Mv69WuzHL9gQTAvvBBA164d8fZuQ0DAANauDcvQx8urKfHx8axY8RteXk3x8mrK++9PArJeo5+cnExQ0Ez69OlOhw6t6Nu3O3PmfEdKSkqm43722XTWrfuDAQP60qFDKwYM6MuWLZvues5388cfvzNkSH+8vVvj6/s4U6a8y6VLlzL0OXHiBG+//QZPPdUZb+/W9OjRlYkTx2XYzTEsbBUBAQN4/PHHeOKJdvj79+PHH3/IdVz3YrFC38fHh6SkJBYvXmxqS0xMJCQkBE9PT8qVKwfA6dOnOXz4cK7m6G4IS7EAACAASURBVNixIzY2NixYsMDUlpaWxsKFC6lQoQINGzYEbi3lyWpLza+++goALy+vXM0v2avnWpp3h7bAs7YzIX8e4cP5O4i9pG04RUQk/7uyZRNH3hzFgecGc+TNUVy5jwIyN1q39sLBwYE1a8IyvbZmTRj16zegXDkXANauDSMh4SY9evTm9dffoHnzlvz884/8978TzZ43PHwL48e/idFoZPjwl2nbtj1TpkwmKmpfpr4//bSQWrXcGDp0OMOHv4SVlRXvvDOWTZv+MvV55513sbe3p2HDxrzzzru88867dO/eM9v5P/roPYKCZlK3bj1eeWUEderU5bvvvmHatCmZ+u7cuZ1PP53G44935sUX/0NiYgLjx7/J5cuXsjjy3S1fvpSJE9/C1taOF1/8D126+LJ69Ur+85/hpuXgSUlJvP76yxw+fIh+/fozcuSb+Pp259SpU8TH31q98PffW5g06W3Kly/Pyy+/xvDhL9OwoSe7d+8yO6acstga/YYNG+Lj48O0adOIi4ujSpUqLFmyhNOnTzNlyr+/sDFjxhAeHp7hgVjR0dGEhoYCsHv3buDfotzd3R1vb28AXFxc8Pf3JygoiISEBBo0aGB6KNaMGTNMD8uKi4ujR48e+Pr6Ur16dVJTU1m7di2bN2+ma9euNGvW7KHk5FHj6GDDC93r0aiWE/N+P8DEoHD6d6yFl0d50409IiIi+cmVLZs4O3cOaf9/D2DyhfOcnTsHgOItWz+UGOzt7Wnd2ot16/7gtddGmf7NjI4+xYEDUbz22mhT3xdffBU7u3+fPdS9e08qVqzMzJlfEhMTg4uLS47n/frr/+Hk5MzXXwdSpMit7dEbN/ZkxIhXcHEpn6HvDz/8nGHeXr36ERDwLIsWzad161sXUDt37sqMGR9ToUJFOnfuete5Dx48wIoVv/H0070YPXrc/x+zL46OxQgNDaFXr37UrFnL1P/48WPMm7eYChUqAuDp2ZTBg/0IC1tFr179cnzOycnJfP3159SsWZvPP//WtMmLm5s7kya9zdKlS+jd+xmOHTvC6dPRzJr1PXXq1DONHzp0uOnvmzZtxNW1Ou+/PzXH898vixX6AB9//DGffvopoaGhXL58GTc3N2bOnEmTJk3uOu7UqVN89tlnGdrSf+7Ro4ep0AcYPXo0JUqUYNGiRYSEhODq6sr06dPp2vXfN1Tx4sVp3749GzduZMmSJaSmplKtWjXGjh2Lv79/Hp6x3MlgMNCqngu1K5UkcNk+Zq+IYuehcwzq4k7xIpl3TBIREckLVzZt5PJff5o97uaRw6QlJ2doS0tM5OycIC7/ud7s45Xweozirc3f8MPb+3H++GM1ERE7adiwMQBr1qzGaDTSoUMnU7/bi+0bN278/4XPW88sOngwKseF/rlz5zh48ACDBg01FfkAzZq1pFq16ty8mfFb+dvnvXLlCqmpqXh4NCYsbJXZ5wqwZctGAPr1ezZDe79+/QkNDWHz5o0ZCv3mzVuainyAmjVrUbRoUU6fjjZr3qiofVy8eIFhw17MsJOjt/fjfPnlZ2zatJHevZ+haNFbN8Nu3LiBmjVrZ7mzpKOjI7GxZ9m7dw/16tU3K47csmihb2dnx5gxYxgzZky2fYKDgzO1tWjRIsMV/ru59fXScIYPH55tn+LFizN16sP7dCWZlSlhz2i/xvwefpKQPw8zITCcIV3caVgz83MQRERELOXOIv9e7Q9Ky5ZtKFKkKH/88XuGQt/Do1GGZwjFxMQQGPgNf/31J1evXslwjKyWLWcnJuYMAJUqVc70WpUqVTlwICpD28aNG/j++0AOHTqQYQfE3H5jHxNzBisrKypWrJShvWLFylhZWXH27JkM7elLl25XrFhxrl41bxOQ9POuUqVqhnaj0UilSpVN81aoUBE/vwHMmfMdixYtoHFjT1q3bssTT/iYPhj17NmHtWvDGD58MOXLV6RZs+Z06NCJZs1amBWTOSxa6Ivczmgw4NOiCvVdSzNz6V4++ymC9o0r0q9DTexsLbsNp4iIFC7FW7fJ1ZX0I2+OIvnC+Uzt1qXLUPnNcXkRWo7Y2dnRpk1b1q9fw+uvv8Hp09EcPHiAkSP/vXiakpLCiBEvcfXqFZ591p+qVathb+/AuXNxvP/+pAf2xPpdu/5h7NiRNGzYmJEjx1CmjBPW1tYsX76U1atXPpA572Q0Zl03PKhzBnjttZF06eLLhg3rCQ/fwieffMTcuUF8++1snJ3LUqpUaWbPXkB4+Ba2bNnEli2b+PXXJTz55FOMGzfhgcSkp0ZIvlOprCPvDGqGT/MqrP8nmkmzwzl8WlucioiI5Tn17IXhjodxGmxtcerZ66HH4u3difPnz7Nz5w7WrFmNlZUVHTp0NL1+5MghTp48wcsvv86AAYNp27Y9zZq1yHDFP6fS1+CfOnUy02snThzP8PO6dWuwtbXlk0++wNe3O61atbnLVeucXeF3cSlPSkoK0dGnMrRHR58iJSWFcuXKZzPy/qSf953nmJaWxqlTJzPNW716TQYNGsqXX87i66+DiI09yy+//Gx63cbGhjZt2jJq1Bh+/PEXevbsw7Jlv2Y6r7yiQl/yJRtrI329a/KGX2OSU1KZEryDXzYcITlFDzoRERHLKd6yNeX8B2Ndugxw60p+Of/BD+1G3Nu1aNGaokWLsmbNatasCaNRoyaUKvXvluBZXdVOS0tj8eKFZs/l5ORErVq1WbHiN65f/3dr9L//3sKxY0cy9DUajRgMBlJT//03+8yZ02zYsC7TcR0cHEy70txNy5a3vn25cyvK9HNJv8E3r7m716VUqdL88stPGR70unbtH8TFxdL6/78VunYtnuQ7lm9Vr14DKysr09KlO3f8MRgM1Khx676C9N178pqW7ki+5l61FJMDWjB/9QF+3XiM3UcuMKxbXVxKF7F0aCIi8ogq3rK1RQr7O9na2uLl1Y5Vq1Zw48Z13nzz7QyvV61ajYoVK/Hll58SFxdL0aJFWbdujdnr1NMNH/4Kb775Oi++OJSuXbtx5coVfv55Ea6u1blx49+bcVu39mLRovmMGvUqjz/emYsXLxISspiKFStz+PDBDMd0c3Nn27ZwFi6ch5OTM+XLV8zyRtVatWrTpYsvS5Ys5urVK3h4NCIiYidhYavw9e1OjRo1c3VO92Jtbc2LL77KBx9M5tVXh9Op0xPExp7lp58WUb16Dbp16wHA9u3bmDHjY9q370iVKlVJTU1h1aoVGAwG2rW7tUnMhx++x9WrV/D0bErZsmU5e/bWcWrVqk21aq4PJv4HclSRPFTE3pph3erSqJYTc1dGMWl2OP28a9G+UQVtwykiIo+0jh2fYNWq5VhZWdGuXYcMr1lbW/PRRzP49NOpBAfPwc7OlrZtO9CrV18GD/Yze66WLVvz3/9+yKxZX/Ptt19SoUIlxo2byF9/reeff7ab+jVp0oyxY99h3rzv+d//PqF8+Qq8+OKrnDlzOlOh//LLr/PRR+8xa9bXJCQk0KWLb7Y70owZM57y5SuwYsVvrFv3B87OZXnuuRcYOHCI2edijq5du2Fra8v8+d/z5ZefUbRoUR5/3IcXXnjV9PTemjVr0bJlKzZt2kBoaAj29vbUrFmLadP+R/36DQDo3LkLv/66hCVLfiI+/iqlS5fB27sTAQHPm7Z8z2uGtAd5V8Ij7vz5+If+xFdn52LExeXuk3pBcPFqAkHL9rH32EU8apRhSBd3Sjjm/hHZhT1fD4JyZh7lyzzKl3mUr3uLiTmOi8utHVOsrY0kJ2sJaE4pX+Z5UPm6/T2cFaPRQJkyjlm/lufRiDxApYrZMaJfI/p3qkXk8Yu8ExjOjgNxlg5LREREJN9RoS8FjtFgoFPTykwY3IzSxe34ImQ3QcsiuZHwcPcwFhEREcnPVOhLgVXRqSjj/ZvyZKuqbNxzholB4Rw4eeneA0VEREQeASr0pUCztjLSq10Nxj7rCcBHC3bw8/rD2oZTREREHnkq9KVQqFWpJJMDmtOmQXmWbT7Oe3O3EX3u2r0HioiIiBRSKvSl0HCwsyagax1e6dmAC1cSeHfO36zedpJUbSwlIiIijyDtoy+FjmdtZ2pUKM7sFVH8EHaQiEPnCHiyLqWK5X4bThEREZGCRlf0pVAq4WjHa7098O/sxsHoy0wI3Ep45FlLhyUiIhagRwZJQXW/710V+lJoGQwG2jeuyKQhzSlbqgjfhO5l1tK9XL+ZZOnQRETkIbGysiYpKdHSYYjkSlJSIlZWuV+Ao0JfCj2X0kV4a6An3b1c2bovlglB4UQdv2jpsERE5CFwdCzJpUtxJCYm6Mq+FBhpaWkkJiZw6VIcjo4lc30crdGXR4KV0Uh3L1caVC/DrKV7mfrDPzzRvDLDezW0dGgiIvIAOTgUBeDy5XNcupRKaqq2X84po9GofJkhr/NlZWVNsWKlTO/h3FChL4+U6hWKM2lIc35ce4hV4SfZf/IyQ7q4U7mso6VDExGRB8TBoSgODkVxdi5GXNxVS4dTYChf5smP+dLSHXnk2NlaMbCzG6/38eBSfAL//f5vVm49oW04RUREpFBRoS+PLI8aTnwxugMNqpfhx7WHmPbDP5y7fMPSYYmIiIjkCRX68kgr4WjHKz0bMKSrO0djrjIxKJzNe2J0w5aIiIgUeCr05ZFnMBho61GByQHNqejsyKzf9vF16F7ib2gbThERESm4VOiL/L+yJR0Y29+TXu2q88+BOCYEbmXv0QuWDktEREQkV1Toi9zGaDTwZKtqjPdvioOdNdMX7WT+6gMkJqVYOjQRERERs1i00E9MTGTq1Kl4eXnh4eFB37592bx58z3HRUREMGnSJHr27En9+vVxc3PLtm9qaiqzZs3C29ubBg0a0K1bN5YvX55l38OHDzN06FAaN25M8+bNGTNmDBcu6Iruo6iqSzEmDm5GpyaV+GP7KSbP+ZvjMflryywRERGRu7FooT927Fi+//57nnrqKd5++22MRiPDhg3jn3/+ueu49evXs3jxYgAqV658174zZsxg2rRpeHl58c4771ChQgVGjBjBypUrM/SLiYnh2Wef5eTJk4wYMYKAgADWrl3L0KFDSUrSWu1Hka2NFf0fr82ofo24kZDMe3O38dumY6Sm6kZdERERyf8MaRbaXiQiIoI+ffowbtw4Bg8eDEBCQgK+vr6ULVuW+fPnZzv23LlzODo6Ym9vz/vvv8/cuXPZv39/pn5nz56lY8eO+Pn58fbbbwO3Hik8YMAAzpw5Q1hYGEbjrc86kyZNIjQ0lJUrV1KuXDkANm3axJAhQ3j//ffp3bu32ed4/nz8Qy8K8+PDGvKznOYr/kYSwav283dULDUrluC5bnUpW9LhIUSY/+g9Zh7lyzzKl3mUL/MoX+ZRvsxjqXwZjQbKlMn6wZ8Wu6K/cuVKbGxs6NOnj6nNzs6O3r17s337dmJjY7Md6+TkhL29/T3nCAsLIykpif79+5vaDAYDfn5+REdHExERYWr//fff8fb2NhX5AK1bt6ZatWqsWLHC3NOTQsbRwYYXutdjWLe6RJ+7xsSgcDbsOq1tOEVERCTfslihHxkZiaurK0WLFs3Q7uHhQVpaGpGRkXkyh6OjI66urpnmANi3bx9w68r/+fPnqV+/fqZjeHh45EksUvAZDAZa1XPh3YDmuLoUY/aKKL4I2c2V64mWDk1EREQkE4sV+nFxcZQtWzZTu7OzM8Bdr+ibM4eTk9M950j/M739zr7nz58nJUW7rsgtZUrYM9qvMX071GT3kfNMCAxn16Fzlg5LREREJANrS0188+ZNbGxsMrXb2dkBt9br58Uctra295wj/c+79b1582ambx/uJbv1Ug+as3Mxi8xbUOU2XwN969G2SWWmz9/OZz9F0KVVNQK61cPezmL/Wz00eo+ZR/kyj/JlHuXLPMqXeZQv8+S3fFmsIrG3t89yN5v0oju9wL7fORITMy+ruHOO9D/v1jcn9wTcSTfj5n/3m6+i1gbGPevJkj+PsHLzMXZEneW5bnWpUaFE3gWZz+g9Zh7lyzzKl3mUL/MoX+ZRvsyjm3Fv4+zsnOXynLi4OIAsl/XkZo5z5zIvqbhzjvQ/09vv7FumTBmsrKzuOx4pnGysjfT1rskbfo1JTkllSvAOftlwhOSUVEuHJiIiIo8wixX67u7uHD16lGvXrmVo37Vrl+n1+1WnTh3i4+M5evRolnPUqVMHgHLlylG6dGn27NmT6RgRERGmfiJ34161FJMDWtCibjl+3XiMKfN2EHPhuqXDEhERkUeUxQp9Hx8fkpKSTA++gltLZ0JCQvD09DRtc3n69GkOHz6cqzk6duyIjY0NCxYsMLWlpaWxcOFCKlSoQMOGDU3tTzzxBGvWrOHs2bOmts2bN3Ps2DF8fHxyNb88eorYWzOsW11efLo+sRevM2l2OGv/idY2nCIiIvLQWWyNfsOGDfHx8WHatGnExcVRpUoVlixZwunTp5kyZYqp35gxYwgPD8/wQKzo6GhCQ0MB2L17NwBfffUVcOubAG9vbwBcXFzw9/cnKCiIhIQEGjRoQFhYGNu2bWPGjBmmh2UBvPDCC6xcuRJ/f38GDBjA9evXCQwMxN3dne7duz/wfEjh0sy9LDUrliBoeSTBq/az69A5hnRxp4Tj/d97IiIiIpITFt0e5OOPP+bTTz8lNDSUy5cv4+bmxsyZM2nSpMldx506dYrPPvssQ1v6zz169DAV+gCjR4+mRIkSLFq0iJCQEFxdXZk+fTpdu3bNML58+fLMmzePDz/8kOnTp2NjY0P79u0ZN25clrvxiNxLqWJ2jOjbkDXbT7F43WHeCQxncBd3PGtn3sZVREREJK8Z0rSm4IHRrjv538PKV/S5a8xaupcTZ+PxalAev061cCig23DqPWYe5cs8ypd5lC/zKF/mUb7Mo113RB5RFZ2KMt6/KU+2qsrGPWeYGBTOgZOXLB2WiIiIFGIq9EUeEmsrI73a1WDss54AfLRgBz+vP6xtOEVEROSBUKEv8pDVqlSSyQHNadOgPMs2H+e9uduIPnft3gNFREREzKBCX8QCHOysCehah1d6NuDClQTenfM3q7edJFW3zIiIiEgeKZh3A4oUEp61nalRoTizV0TxQ9hBIg6dI+DJupQqpm04RURE5P7oir6IhZVwtOO13h74d3bjYPRlJgRuJTzy7L0HioiIiNyFCn2RfMBgMNC+cUUmDWlO2VJF+CZ0LzOX7uX6zSRLhyYiIiIFlAp9kXzEpXQR3hroSXcvV8L3xTIhKJzI4xctHZaIiIgUQCr0RfIZK6OR7l6uvDWwCTZWRqb98A+L1hwkKVnbcIqIiEjOqdAXyaeqVyjOpCHNad+4IqvCT/Lf7//mZGy8pcMSERGRAkKFvkg+ZmdrxcDObrzex4Mr15P47/d/s3LrCW3DKSIiIvekQl+kAPCo4cS7Q5vToHoZflx7iGk//MO5yzcsHZaIiIjkYyr0RQqI4kVseaVnA4Z0dedozFUmBoWzeU8Mabq6LyIiIllQoS9SgBgMBtp6VODdgOZUdHZk1m/7+Dp0L/E3tA2niIiIZKRCX6QAci7pwNj+nvRqV51/DsQxIXAre49esHRYIiIiko+o0BcpoIxGA0+2qsZ4/6Y42FkzfdFO5q8+QGJSiqVDExERkXxAhb5IAVfVpRgTBzejU5NK/LH9FJPn/M3xmKuWDktEREQsTIW+SCFga2NF/8drM6pfI24kJPPe3G38tukYqam6UVdERORRpUJfpBCp51qad4e2wLO2MyF/HuHD+TuIvaRtOEVERB5FKvRFChlHBxte6F6PYd3qEn3uGhODwtmw67S24RQREXnEqNAXKYQMBgOt6rnwbkBzXF2KMXtFFF+E7ObK9URLhyYiIiIPiQp9kUKsTAl7Rvs1pm+Hmuw+cp4J321l16Fzlg5LREREHgIV+iKFnNFgwKdFFSYMakbxorZ89lMEc1dGkZCobThFREQKMxX6Io+ISmUdeWdQM3yaV2H9ztNMnB3O4dOXLR2WiIiIPCAq9EUeITbWRvp61+QNv8akpKQyJXgHv2w4QnJKqqVDExERkTxm0UI/MTGRqVOn4uXlhYeHB3379mXz5s05Gnv27Flee+01mjZtiqenJy+99BInT57Mst/o0aNp0aIFDRs2pG/fvvz111+Z+o0dOxY3N7dM//Xt2/e+z1Mkv3GvWorJAS1oUbccv248xpR5O4i5cN3SYYmIiEgesrbk5GPHjuX333/H39+fqlWrsmTJEoYNG0ZwcDCNGzfOdty1a9fw9/fn2rVrvPDCC1hbWzNnzhz8/f355ZdfKFGiBABXrlzBz8+Py5cv4+/vj5OTEytWrOD5558nMDCQVq1aZTiug4MDkydPztBWunTpvD9xkXygiL01w7rVpVEtJ+aujGLS7HD6edeifaMKGAwGS4cnIiIi98lihX5ERATLli1j3LhxDB48GICnn34aX19fpk2bxvz587Mdu2DBAo4fP05ISAh169YFoG3btnTr1o05c+bw2muvAbBw4UKio6OZN28ezZo1A8DPz4++ffvy4YcfEhoamuG41tbWdO/e/QGcrUj+1cy9LDUrliBoeSTBq/az69A5hnRxp4SjnaVDExERkftgsaU7K1euxMbGhj59+pja7Ozs6N27N9u3byc2NjbbsatWraJRo0amIh+gRo0atGrVihUrVpjaduzYgbOzs6nIBzAajXTp0oWoqCiOHDmS6dgpKSnEx8ff7+mJFCilitkxom9D+neqReTxi7wTGM6OA3GWDktERETug8UK/cjISFxdXSlatGiGdg8PD9LS0oiMjMxyXGpqKvv376d+/fqZXmvQoAHHjh3jxo0bACQlJWFvb5+pX3rbvn37MrRfu3aNJk2a0KRJE1q0aMGUKVNISEjI1fmJFDRGg4FOTSszcXAzyhS354uQ3QQti+RGQrKlQxMREZFcsNjSnbi4OMqVK5ep3dnZGSDbK/qXLl0iMTHR1O/OsWlpacTFxVGlShVcXV3ZvHkzMTExuLi4mPpt37490xzOzs4899xz1KlTh9TUVNauXcucOXM4fPgw33333X2dq0hBUsGpKG/7NyH0r6Ms33KcqBMXec63LrUrl7R0aCIiImIGixX6N2/exMbGJlO7nd2tdcHZXUlPb7e1tc127M2bNwHo3bs3Cxcu5LXXXmPs2LE4OTmxfPlyVq9enaEfwKhRozIcy9fXl3LlyhEYGMjGjRtp06aNuadImTKOZo/JC87OxSwyb0GlfGXthd6NeKxJZT5ZsIOPF+ygl3ct/J5wB5Qzcylf5lG+zKN8mUf5Mo/yZZ78li+LFfr29vYkJSVlak8v5NOL9jultycmJmY7Nn1pjru7O9OmTWPixIk888wzwK0r92+99RaTJk2iSJEid40xICCAwMBANm/enKtC//z5eFJT08wedz+cnYsRF3f1oc5ZkClfd+fsaMuEQU354Y+DLP7jIFv3nGGMfzMcrLQrT07pPWYe5cs8ypd5lC/zKF/msVS+jEZDtheXLVboOzs7Z7k8Jy7u1g2AZcuWzXJcyZIlsbW1NfW7c6zBYMiwrMfHxwdvb2+ioqJITU2lbt26hIeHA1CtWrW7xujk5ISNjQ2XL+vpofLocrCzJqBrHRrVdGLOiihGzFhPr/Y16NikEkZtwykiIpJvWexmXHd3d44ePcq1a9cytO/atcv0elaMRiO1a9dmz549mV6LiIigatWqODg4ZGi3tbXFw8ODRo0aYWtry6ZNm7C1tcXT0/OuMcbExJCUlKS99EUAz9rO/HdoczxqOfND2EFmLNrJxau6WV1ERCS/slih7+PjQ1JSEosXLza1JSYmEhISgqenp+lG3dOnT3P48OEMYzt37szOnTsz7Jpz5MgRtmzZgo+Pz13nPXbsGAsXLqRHjx4UL14cuLXkJ6stNb/66isAvLy8cneSIoVMCUc7JgxtgX9nNw5GX2ZC4FbCI89aOiwRERHJgsWW7jRs2BAfHx+mTZtm2iVnyZIlnD59milTppj6jRkzhvDwcPbv329q69+/P4sXL+b5559nyJAhWFlZMWfOHJydnU0P3wJITk6me/fudO7cmfLly3Pq1CkWLlxIhQoVGD16tKlfXFwcPXr0wNfXl+rVq5t23dm8eTNdu3bNsA+/yKPOYDDQvnFF3KuWYtbSfXwTupedh84x4PHaFLHPfIO9iIiIWIbFCn2Ajz/+mE8//ZTQ0FAuX76Mm5sbM2fOpEmTJncd5+joSHBwMB988AFfffUVqamptGjRgrfffptSpUqZ+hmNRmrVqsXPP//M+fPncXJy4umnn+aVV16hWLF/74ouXrw47du3Z+PGjSxZsoTU1FSqVavG2LFj8ff3f2DnL1KQuZQuwlsDPflt03GWbjzGgZOXGPpkXepULXXvwSIiIvLAGdLS0h7utjCPEO26k/8pX+bLKmdHTl9h1tK9xF68wRPNK9PzserYWFtZKML8Re8x8yhf5lG+zKN8mUf5Mk9+3HXHYmv0RaTwqF6hOJOGNKd944qsCj/Jf7/fxsnYzPe9iIiIyMOjQl9E8oSdrRUDO7vxeh8PrlxP4r/f/83KrSdI1ZeGIiIiFqFCX0TylEcNJ94d2pwG1cvw49pDTPvhH85dvmHpsERERB45KvRFJM8VL2LLKz0bMKSrO0djrjIxKJzNe2LQLUEiIiIPjwp9EXkgDAYDbT0q8G5Acyo6OzLrt318HbqX+BtJlg5NRETkkaBCX0QeKOeSDozt70mvdtX550AcEwK3svfoBUuHJSIiUuip0BeRB85oNPBkq2qM92+Kg5010xftZP7qAyQmpVg6NBERkUJLhb6IPDRVXYoxcXAzOjWpxB/bTzF5zt8cj9EezSIiIg+CCn0Reahsbazo/3htRvVrxI2EZN6bu43fNh176A+Xm1Do5QAAIABJREFUExERKexU6IuIRdRzLc27Q1vgWduZkD+P8OH8HcRe0jacIiIieUWFvohYjKODDS90r8ewbnWJPneNiUHhbNh1WttwioiI5AEV+iJiUQaDgVb1XHg3oDmuLsWYvSKKL0J2c+V6oqVDExERKdBU6ItIvlCmhD2j/RrTt0NNdh85z4TvtrLr0DlLhyUiIlJgqdAXkXzDaDDg06IKEwY1o3hRWz77KYK5K6NISNQ2nCIiIuZSoS8i+U6lso68M6gZPs2rsH7naSbODufw6cuWDktERKRAUaEvIvmSjbWRvt41ecOvMSkpqUwJ3sEvG46QnJJq6dBEREQKBBX6IpKvuVctxeSAFrSoW45fNx5jyrztxFy4bumwRERE8j0V+iKS7xWxt2ZYt7q8+HR9Yi/eYFJQOGt3nNI2nCIiIndhbekARERyqpl7WWpWLEHQ8kiCfz/ArsPnGdLFnRKOdpYOTUREJN/RFX0RKVBKFbNjRN+G9O9Ui8jjF3knMJwdB+IsHZaIiEi+kyeFfnJyMqtWreLHH38kLk7/4IrIg2U0GOjUtDITBzejTHF7vgjZTdCySG4kJFs6NBERkXzD7KU7H3/8MVu3buXnn38GIC0tjSFDhrBt2zbS0tIoWbIkP/74I1WqVMnzYEVEblfBqShv+zfh141HWbb5OFEnLvKcb11qVy5p6dBEREQszuwr+hs2bKBp06amn9esWcPff//N0KFDmT59OgAzZ87MuwhFRO7C2spIz8dqMPbZ/2Pv3sOqLvP9/z/XgrU4n1minA8KKAgiHkvLzJIcLXPMRtuR1tQ0fWeP4+z5fdOaqcbmZ1M5k+5mt+eqLMqxk4m61TxUOlbmISUlBFEBTUURQVCRk671/QNdewhIl6IL8PW4rq7y/nzue92f9/WR3tzrPvTHYIAX38thycYibcMpIiI3PIdH9I8dO0ZUVJT9zxs2bCA8PJzf/e53AOzbt48VK1a0Xw9FRC5Dr3B/nps2iA8+38eqzQf5rriCR8clERbs5eyuiYiIOIXDI/qNjY24uv7v7wdbt27lpptusv85IiLisufpNzQ08PLLLzNs2DBSUlKYNGkSmzdvvqy6ZWVlTJ8+nQEDBtC/f3+eeOIJDh061Op9v/vd7xg8eDCpqalMmjSJr776qtU2i4qKeOSRR0hLS2PQoEE8+eSTVFZWXlZ/RMT5PNxcmTamN7+a0JfKU/XMzvqGT7cfwqptOEVE5AbkcKLfvXt3vv32W6Bp9P7QoUMMHDjQfr2iogJPT8/LamvmzJm888473H333Tz99NMYjUYeffRRe/ttqampITMzkx07dvD444/z61//mvz8fDIzM6murrbfd+rUKSZPnsyGDRuYMmUK//f//l/MZjOPPfZYi18ojh07xgMPPMChQ4eYMWMGDz/8MBs2bOCRRx6hsbHxcsMjIh1A/3gLzz8yiN5RAbz/2T5e+XAnJ0/XO7tbIiIi15XDU3d+8pOf8Nprr1FZWcm+ffvw9vbm1ltvtV8vKCi4rIW4ubm5rFq1ilmzZjF16lQAxo8fz9ixY5k7dy6LFi1qs+57773HwYMHyc7Opk+fPgAMHz6ccePGkZWVxfTp0wH44IMPOHLkCP/4xz/sv4xMnjyZSZMm8ec//5nly5fb2/z73/9OfX09CxcuJCQkBICUlBSmTZvG8uXLmThxomOBEhGn8vN2Y/rEFDbuLOWD9ft4ZsFWHhydwKDeIc7umoiIyHXh8Ij+L37xC+6991527tyJwWDgxRdfxNfXF4DTp0+zfv16hg4desl21qxZg8lk4r777rOXubm5MXHiRHbs2MHx48fbrLt27Vr69etnT/IB4uLiGDp0KKtXr7aX5eTkYLFYmn3jYDQaueuuu9izZw/FxcX28nXr1jFy5Eh7kg9w0003ER0d3axNEek8DAYDI9LCeG7aILoFePL35bt5fcVuztbpWzoREen6HB7RN5vNzJkzp9VrXl5efPXVV7i7u1+ynYKCAmJiYvDyar5QLiUlBZvNRkFBAd26dWtRz2q1UlhYyP3339/iWt++fdm0aRO1tbV4eHjQ2NjYal8uluXn5xMbG0tZWRkVFRUkJye3uDclJYVNmzZd8nlEpOPqHujJUw/2Z+XXB1mx6QB7D1XxyE/60DsqwNldExERuWba9WTcc+fO4ePjg8lkuuS95eXlrSbyFosFoM0R/aqqKhoaGuz3/bCuzWazLwaOiYmhtLSUY8eONbtvx44dzT7j4r/barOiooLz589f8plEpONyMRq5Z1gMTz2YjsnFyNz3v+XD9ftoPKe/2yIi0jU5PKK/ceNGcnNz+fd//3d72aJFi/jLX/5CXV0dd911F3/+858vmezX1dW1eo+bmxsA9fWtL5y7WG42m9usW1dXB8DEiRP54IMPmD59OjNnziQ4OJhPPvmETz/9tNl9l9vmD799uJSgIG+H7m8vFouPUz63s1K8HNeZY2ax+JCaGMJbK3ez+usDFB6q5rdT+hMT6ndNP1Mun+LlGMXLMYqXYxQvx3S0eDmc6C9YsICgoCD7n4uKipgzZw4RERGEh4fzySef0LdvX/sC27a4u7u3upvNxaT7YoL9QxfLGxoa2qx7cWpOYmIic+fO5dlnn+VnP/sZ0DRC/9RTT/Hcc8/ZdwdypE1HVFScwWq9vtv6WSw+lJefvq6f2ZkpXo7rKjG775ZYEsJ8eeuTPfx23kbuvSWW0QMjMRoN7fo5XSVe14vi5RjFyzGKl2MUL8c4K15Go6HNwWWHp+4UFxc3m8v+ySef4Obmxscff8ybb77JmDFjWLZs2SXbsVgsrU7PuTjtprVpPQD+/v6YzeZW9+ovLy/HYDA0m4KTkZHBl19+yeLFi/nwww9Zv349ERERAERHRzf7rLbaDAoKwsXF5ZLPJCKdS0pcMLMfGUTf2CAWbyji5fe/5UR1rbO7JSIi0i4cTvSrq6sJCPjfBWxff/01Q4YMwdu76TeJQYMGcfjw4Uu2k5iYSElJCTU1Nc3Kd+3aZb/eaoeNRuLj48nLy2txLTc3l6ioKDw8PJqVm81mUlJS6NevH2azma+//hqz2Uz//v0BCAkJITAwsM02e/fufcnnEZHOydfTzK8m9GXamEQOlJ3m2be28XXeUWw6ZEtERDo5hxP9gIAASktLAThz5gzfffcdAwYMsF8/d+7cZS1czcjIoLGxkcWLF9vLGhoayM7Opn///vZtLktLSykqKmpWd/To0ezcuZP8/Hx7WXFxMVu2bCEjI+NHP/fAgQN88MEH3HvvvfZtQQHuvPNO1q9fT1lZmb1s8+bNHDhw4JJtikjnZjAYGJ4SyuyHBxFm8ebNlQX89/LdnKnVNpwiItJ5OTxHv1+/fnzwwQf07NmTL774gvPnz3PLLbfYrx88eLDNaTf/KjU1lYyMDObOnUt5eTmRkZEsXbqU0tJSXnjhBft9Tz75JNu2baOwsNBeNmXKFBYvXsxjjz3GtGnTcHFxISsrC4vF0mxtwLlz57jnnnsYPXo0PXr04PDhw3zwwQeEhobyu9/9rll/Hn/8cdasWUNmZib/9m//xtmzZ1mwYAGJiYncc889joZJRDohi78HM6f0Z/XWgyz7soT9h5u24UyKCXR210RERBzmcKL/61//mszMTH7zm98AcO+999KzZ08AbDYbn332GYMHD76stl566SXmzZvH8uXLqa6uJiEhgddff5309PQfreft7c3ChQuZM2cOr732GlarlcGDB/P00083m1ZkNBrp1asXS5YsoaKiguDgYMaPH8+vfvUrfHyar4ru0aMH//jHP/jzn//MX/7yF0wmEyNGjGDWrFmt7sYjIl2T0WjgJ0OjSY4J4vUVu/nLhzu5PT2c+0bEYTZprY6IiHQeBtsVTEStqqoiJycHHx+fZqfOVldXs2zZMgYPHtzmHPsbiXbd6fgUL8fdSDFraDzPxxuL+Gz7YXoEefLYuCSiuju2ddqNFK/2oHg5RvFyjOLlGMXLMR1x1x2HR/ShaeebkSNHtij38/PjoYceupImRUQ6HLPJhSmj4kmNC2bBqnz+9O527hkWw5ghUe2+DaeIiEh7u6JEH+D777/n888/59ChQwBERERw++23ExkZ2W6dExHpCJJiApn9yGD+sa6Q7C+KyS2q4Ofj+tDN3+PSlUVERJzkihL9efPm8cYbb7TYXefll1/mF7/4BdOnT2+XzomIdBTeHiZ+cXcSqT2D+ce6vTz71jam3N6LYSk9MBg0ui8iIh2Pw4n+xx9/zN///nfS0tL4+c9/Tq9evQDYt28fCxYs4O9//zsRERFMmDCh3TsrIuJMBoOBoUndiQ/3Z8GqfN5evYed+0/w0F2J+Hpq0b6IiHQsDi/GnTBhAiaTiUWLFuHq2vz3hHPnzvHAAw/Q2NhIdnZ2u3a0M9Ji3I5P8XKcYtbEarOxbtshsr8owtPNlWljepPaM7jFfYqXYxQvxyhejlG8HKN4OaYjLsZ1+MCsoqIixowZ0yLJB3B1dWXMmDEtDrgSEelqjAYDGYMjeeahgfh6mZn/cS7vrtlDfcOlDwwUERG5HhxO9E0mE2fPnm3zek1NDSaT6ao6JSLSWYR38+YPDw0kY1AkG3eW8uzb2ygqrXZ2t0RERBxP9Pv27cuHH37IiRMnWlyrqKjgo48+IjU1tV06JyLSGZhcjUwa2ZP/b3Ia589beWFhDsu+LObceauzuyYiIjcwhxfjPvHEE0ydOpUxY8bw05/+1H4q7v79+8nOzqampoa5c+e2e0dFRDq6xKgA/vjwYBZ9upf/2XSA74orePKhQWiZroiIOIPDif7AgQN59dVXef7553n77bebXQsNDeXFF19kwIAB7dZBEZHOxNPdlUfH9aFfr2DeXbOHX//ln9x/Wxwj0sK0DaeIiFxXV7SP/siRIxkxYgR5eXkcPnwYaDowKykpiY8++ogxY8bwySeftGtHRUQ6k4GJ3egZ5sc/Pt3LwnV72bm/gofHJOLn7ebsromIyA3iik/GNRqNpKSkkJKS0qz85MmTlJSUXHXHREQ6uwAfN557dCgfri1g8T+L+MOCbTyUkUh6gsXZXRMRkRuAw4txRUTk8hmNBkYNiODZqQMJ8nXnv5Z+x1urCqitP+fsromISBenRF9E5DoIDfbi6cx0xt4Uxaa8ozz71jb2HqpydrdERKQLU6IvInKduLoYmXBLHDMf6I/BAC++l8OSjUXahlNERK4JJfoiItdZr3B/nps2iGF9e7Bq80H+9O52jpyocXa3RESki7msxbg/3Ebzx+Tk5FxxZ0REbhQebq5MG9Ob1J7BZK3ew+ysb5g4Io7b08MxahtOERFpB5eV6L/44osONaq9okVELk//eAtxYX68/UkB73+2j9z9J3j4J30I8NE2nCIicnUuK9F/9913r3U/RERuWH5eZqZPTGHjzlI+WL+PZxZs5cHRCQzqHeLsromISCd2WYn+oEGDrnU/RERuaAaDgRFpYSRGBfDGinz+vnw3O/ef4N/uiMfT3eTs7omISCekxbgiIh1I90BPnnqwP/cMi2Fb/nGeeWsbBQdPOrtbIiLSCSnRFxHpYFyMRu4ZFsNTD6ZjcjEy9/1v+XD9PhrPnXd210REpBNRoi8i0kHFhvry3LRBjEgLY+22Qzz/znYOHT/j7G6JiEgnoURfRKQDczO78ODoBH5zXwqnzjby/DvfsHrrQaxWm7O7JiIiHZwSfRGRTiAlLpjZjwyib2wQizcU8fL733KiutbZ3RIRkQ7MqYl+Q0MDL7/8MsOGDSMlJYVJkyaxefPmy6pbVlbG9OnTGTBgAP379+eJJ57g0KFDLe47ffo0L774InfeeScpKSmMHDmSZ555hrKysmb3vfrqqyQkJLT45+abb26XZxURuVq+nmZ+NaEv08YkcqDsNM++tY2v845is2l0X0REWrqs7TWvlZkzZ7Ju3ToyMzOJiopi6dKlPProoyxcuJC0tLQ269XU1JCZmUlNTQ2PP/44rq6uZGVlkZmZybJly/Dz8wPAarXyyCOPsG/fPiZPnkxMTAwlJSW8//77bNmyhZUrV2I2m5u1PXv2bNzd3e1//tf/FhFxNoPBwPCUUBIjA3hjZT5vrixg5/4KMkcn4O2hbThFROR/OS3Rz83NZdWqVcyaNYupU6cCMH78eMaOHcvcuXNZtGhRm3Xfe+89Dh48SHZ2Nn369AFg+PDhjBs3jqysLKZPnw7Ad999x65du3jmmWd44IEH7PVDQ0N5/vnnycnJYciQIc3avuuuu/D19W3npxURaV8Wfw9mTunP6q0HWfZlCfsOV/HIT3qTHBPk7K6JiEgH4bSpO2vWrMFkMnHffffZy9zc3Jg4cSI7duzg+PHjbdZdu3Yt/fr1syf5AHFxcQwdOpTVq1fby86cadqdIiio+f/4goODgdZH6202G2fOnNFX4SLS4RmNBn4yNJrfZw7A082Vv364i0Xr9lLfqG04RUTEiYl+QUEBMTExeHl5NStPSUnBZrNRUFDQaj2r1UphYSHJycktrvXt25cDBw5QW9u0QC0pKQlPT0/mz5/P5s2bKSsrY/PmzcyfP5/BgweTmpraoo0RI0aQnp5Oeno6s2bNoqqqqh2eVkTk2onq7sOzUwcyakA4n+ccZnbWNxw8dtrZ3RIRESdz2tSd8vJyQkJCWpRbLBaANkf0q6qqaGhosN/3w7o2m43y8nIiIyPx9/fnlVde4fe//719ehDAbbfdxrx58zAYDPYyX19fHnzwQVJTUzGZTGzZsoUPP/yQ/Px8Fi9e3GIuv4hIR2I2uTBlVDypccEsWJXPn97dzj3DYhgzJAqj0XDpBkREpMtxWqJfV1eHydRy4ZibmxsA9fX1rda7WN5a4n2xbl1dnb0sMDCQ5ORk0tLSiIuLY8+ePbz55ps89dRT/PWvf7Xf99BDDzVrKyMjg169ejF79myWLVvGpEmTHHxCCArydrhOe7BYfJzyuZ2V4uU4xcwx1zNeIyw+pCf34L+X5JL9RTEF31fx2yn96R7kdenKHYTeL8coXo5RvByjeDmmo8XLaYm+u7s7jY2NLcovJvIXk/Yfulje0NDQZt2Lc+8PHTpEZmYmc+fOZdSoUQCMGjWKsLAwZs6cyU9/+tMf3T5z8uTJvPzyy2zevPmKEv2KijPX/VAbi8WH8nJ9ZX+5FC/HKWaOcVa8po6Op3eEHwvX7eVXczcw5fZeDEvp0eybzI5I75djFC/HKF6OUbwc46x4GY2GNgeXnTZH32KxtDo9p7y8HIBu3bq1Ws/f3x+z2Wy/74d1DQaDfVpPdnY2DQ0N3Hrrrc3uGzlyJAA5OTk/2kej0UhISAjV1dWXfiARkQ7EYDAwJKk7sx8eREx3H95evYe/ZX/HqbMtB0lERKRrclqin5iYSElJCTU1Nc3Kd+3aZb/eGqPRSHx8PHl5eS2u5ebmEhUVhYeHBwAVFRXYbLYWO+icO3eu2b/b0tjYyNGjRwkICLi8hxIR6WCC/Nz53eQ0Jt3Wk++KK3jmza3s2n/C2d0SEZHrwGmJfkZGBo2NjSxevNhe1tDQQHZ2Nv3797cv1C0tLaWoqKhZ3dGjR7Nz507y8/PtZcXFxWzZsoWMjAx7WXR0NFartdmWmwArV64EaLY9Z2VlZYs+LliwgPr6eoYPH34VTyoi4lxGg4GMwZE889BAfL3MzP84l3fX7KG+Qdtwioh0ZQabEzeMnz59Op9//jkPPfQQkZGRLF26lLy8PN555x3S09MBePDBB9m2bRuFhYX2emfOnOHee++ltraWadOm4eLiQlZWFjabjWXLltlH4E+ePMm4ceOoqqpi8uTJ9OzZk927d/Pxxx/Ts2dPlixZYl8QnJqaypgxY4iPj8dsNrN161bWrl1Leno67777Lq6uji9n0Bz9jk/xcpxi5piOFq/Gc1aWflHM2m3fYwnw4NFxfYgL9XN2t+w6Wrw6OsXLMYqXYxQvx3TEOfpOW4wL8NJLLzFv3jyWL19OdXU1CQkJvP766/Ykvy3e3t4sXLiQOXPm8Nprr2G1Whk8eDBPP/10s2k2AQEBLFmyhPnz57N+/Xref/99/P39mThxIjNmzGi268+4cePIyclhzZo1NDY2EhYWxhNPPMEvfvGLK0ryRUQ6IpOrkUkje5ISF8SCVfm8sDCHsTdFMfamaFxdnPYlr4iIXANOHdHv6jSi3/EpXo5TzBzTkeN1tu4ciz7dy+bdx4jp4cOj45LoHujp1D515Hh1RIqXYxQvxyhejumII/oavhERuUF5urvy6Lg+/HJ8MsdP1vLcW9vYkHO4xQYGIiLSOSnRFxG5wQ1M7MbsRwbTK8Kfhev2Mm9xLtVnWj+0UEREOg8l+iIiQoCPGzMmpTJlVC/2fH+SPyzYxo7ClueViIhI56FEX0REgKZtOEcNiODZqQMJ8nXnv5Z+x1urCqit//EzR0REpGNSoi8iIs2EBnvxdGY6Y2+KYlPeUZ59axt7D1U5u1siIuIgJfoiItKCq4uRCbfEMfOB/hgM8OKiHD7+ZxHnzlud3TUREblMSvRFRKRNvcL9eW7aIIal9OCTLQf50zvbOVJ+xtndEhGRy6BEX0REfpSHmyvTxvTmVxP6Unm6nj9mbefTbw5h1TacIiIdmo58FRGRy9I/3kJcmB9vf1LA+5/vI7foBA//pA8BPm7O7pqIiLRCI/oiInLZ/LzMTJ+YQmZGAvuOVPPMgq1sKyhzdrdERKQVSvRFRMQhBoOBEf3C+OO0QYQEevL35bt5fcVuztY1OrtrIiLyL5Toi4jIFQkJ9GTWv/XnnmExbMs/zjNvbaPg4Elnd0tERC5Qoi8iIlfMxWjknmExPPVgOiYXI3Pf/5YP1++j8dx5Z3dNROSGp0RfRESuWmyoL89NG8SItDDWbjvE8+9s59BxbcMpIuJMSvRFRKRduJldeHB0Ar+5L4VTZxt5/p1vWL31IFartuEUEXEGJfoiItKuUuKCmf3IIPrGBrF4QxEvv/8tJ6prnd0tEZEbjhJ9ERFpd76eZn41oS/TxiRyoOw0z761ja/zjmLTIVsiIteNEn0REbkmDAYDw1NCmf3wIMIs3ry5soD/Xr6bM7XahlNE5HpQoi8iIteUxd+DmVP689NbY/l2bzl/WLCVvJIKZ3dLRKTLU6IvIiLXnNFo4CdDo/l95gA83Vz564e7WLRuL/WN2oZTRORacXV2B0RE5MYR1d2HZ6cO5OONRXy2/TD5Byt5dFwfjlacJXtjEZWn6gn0dWPCrXEMTeru7O6KiHRqSvRFROS6MptcmDIqntS4YBasyuf5rO0YjQbOX9iGs+JUPe+s3gOgZF9E5Cpo6o6IiDhFUkwgsx8ZjMnVaE/yL2o4ZyV7Y5GTeiYi0jUo0RcREafx9jDRcM7a6rWKU/UcrajRlpwiIldIU3dERMSpgnzdqDhV3+q1p9/YSpCvG0kxgSTHBNE7OgAvd9N17qGISOfk1ES/oaGB+fPns3z5ck6dOkViYiIzZsxg6NChl6xbVlbGnDlz2LRpE1arlSFDhjBr1iwiIiKa3Xf69Glee+01Pv/8c44dO0ZwcDDDhg3j//yf/0NISMgVtSkiIu1nwq1xvLN6T7ORfbOrkXtvjcXN1YW8kkq+2XOcL3YdxWCA2B6+TYl/bBAxPXxwMerLaRGR1hhsTvxO9Le//S3r1q0jMzOTqKgoli5dSl5eHgsXLiQtLa3NejU1NUyYMIGamhqmTp2Kq6srWVlZGAwGli1bhp+fHwBWq5Wf/exn7Nu3j8mTJxMTE0NJSQnvv/8+FouFlStXYjabHWrTERUVZ7Bar294LRYfystPX9fP7MwUL8cpZo5RvC7P5t3HfnTXnXPnrRSXniKvpJLdJZUcOHoKG+Dp5kqf6AD7iH+Qn7vzHsIJ9H45RvFyjOLlGGfFy2g0EBTk3eo1p43o5+bmsmrVKmbNmsXUqVMBGD9+PGPHjmXu3LksWrSozbrvvfceBw8eJDs7mz59+gAwfPhwxo0bR1ZWFtOnTwfgu+++Y9euXTzzzDM88MAD9vqhoaE8//zz5OTkMGTIEIfaFBGR9jc0qTtDk7q3+T9KVxcj8RH+xEf4M+GWWM7UNpJ/oJK84krySirYXlgOQI8gT3vSnxDpj5vJ5Xo/iohIh+G0RH/NmjWYTCbuu+8+e5mbmxsTJ07klVde4fjx43Tr1q3VumvXrqVfv372hBwgLi6OoUOHsnr1antSfubMGQCCgoKa1Q8ODgbA3f1/R34ut00REXE+bw8Tg3qHMKh3CDabjdITNeSVVJJXUsnGnaV8tv0wri4GeoX7kxzblPiHW7wwGAzO7rqIyHXjtES/oKCAmJgYvLy8mpWnpKRgs9koKChoNdG3Wq0UFhZy//33t7jWt29fNm3aRG1tLR4eHiQlJeHp6cn8+fPx8/MjNjaW4uJi5s+fz+DBg0lNTXW4TRER6VgMBgNhFm/CLN6MHhRJQ+N59h6uIq+4aZrP4g1FLN5QhJ+X+cJofyB9YgLx9TQ7u+siIteU0xL98vLyFothASwWCwDHjx9vtV5VVRUNDQ32+35Y12azUV5eTmRkJP7+/rzyyiv8/ve/t08PArjtttuYN2+efWTHkTZFRKRjM5tcSI4JIjmm6dvcylN17C6pZPeBSnbtP8HXeccwAJHdfUi+kPjHhfnh6qJFvSLStTgt0a+rq8NkarlFmpubGwD19a1vtXax/OIi2tbq1tXV2csCAwNJTk4mLS2NuLg49uzZw5tvvslTTz3FX//61ytq83K1tTDiWrNYfJzyuZ2V4uU4xcwxipdj2jteFosPCXEWJgDnrTaKDleRU3icnD3HWb31e1ZtPoiHmwspPS2kJXQjLcFCaLBzfn5fCb1fjlG8HKN4Oaajxctpib67uzuNjY0tyi8m3RcT7B+6WN7Q0NBm3Ytz7w8dOkRmZiZz5S8AAAAgAElEQVRz585l1KhRAIwaNYqwsDBmzpzJT3/6U26++WaH2nSEdt3p+BQvxylmjlG8HHM94hXg4crt/UK5vV8oZ+vOUXDwJLtLKsgrqWTr7mNN/fB3v/CtQCCJUQF4uHXMY2f0fjlG8XKM4uUY7brzLywWS6vTc8rLm3ZOaGshrr+/P2az2X7fD+saDAb7FJzs7GwaGhq49dZbm903cuRIAHJycrj55psdalNERLoOT3dX0hMspCc0TdM8frK2aVFvcQVf5x1jw7dHcDEaiAv1JSm2KfGP6u6DUYt6RaQTcFqin5iYyMKFC6mpqWm2IHfXrl32660xGo3Ex8eTl5fX4lpubi5RUVH2RbMVFRXYbLYWx6efO3eu2b8daVNERLomg8FASKAnIYGe3J4eTuM5K/uPVJNXUsHukkqWflHM0i+K8fYwkRQTSFJ0IEkxgQT4tP4NtIiIszlt5VFGRgaNjY0sXrzYXtbQ0EB2djb9+/e3L9QtLS2lqKioWd3Ro0ezc+dO8vPz7WXFxcVs2bKFjIwMe1l0dDRWq5XVq1c3q79y5UqAZltpXm6bIiJyYzC5GukdFcB9I3ry3LRBvPLvw3h0bB/6xgZScKCStz4p4D/+axPPLNjKR+v3s/tAJY3nzju72yIidk49GXf69Ol8/vnnPPTQQ0RGRtpPxn3nnXdIT08H4MEHH2Tbtm0UFhba6505c4Z7772X2tpapk2bhouLC1lZWdhsNpYtW0ZAQAAAJ0+eZNy4cVRVVTF58mR69uzJ7t27+fjjj+nZsydLliyxLwi+3DYdoTn6HZ/i5TjFzDGKl2M6S7ysNhuHj5+xT/PZd7ia81YbZlcj8ZH+9vn9PYI8r+ne/Z0lXh2F4uUYxcsxHXGOvlMT/fr6eubNm8eKFSuorq4mISGB3/72t9x00032e1pL9AGOHTvGnDlz2LRpE1arlcGDB/P0008TERHR7L6ysjLmz5/P1q1bKSsrw9/fn5EjRzJjxowWyfvltnm5lOh3fIqX4xQzxyhejums8aprOEfh91X2Q7vKKs8CEOjrRlJ0IMmxQfSJDsDLveVuc1ejs8bLWRQvxyhejlGif4NRot/xKV6OU8wco3g5pqvE60RVLXkHKtldXEn+wZPU1p/DYIDYHr4XDu0KIibUBxfj1c2g7Srxul4UL8coXo7piIl+x9wvTEREpBML9vdgRL8wRvQL47zVSnHpKfKKm0b7V2w6wP9sOoCnmyu9owPsp/UG+2nTBxFpX0r0RUREriEXo5Fe4f70Cvfn3ltiOVPbSP6BpqR/d0klOwqbtnbuHujZdFJvbCAJEQG4mV2c3HMR6eyU6IuIiFxH3h4mBvUOYVDvEGw2G6UVZ9ld3HRg18ZdpXy24zCuLgZ6hfuTHNO0hWdEN+9ruqhXRLomJfoiIiJOYjAYCAv2IizYizsHRdLQeJ69h6vYfWFR7+J/FrH4n0X4eZnpE9002p8UHYivl9nZXReRTkCJvoiISAdhNrlc2JoziPuBk6fr7Qd25RadYPPuYwBEhfiQHBvIzf3CCfY24eritGNxRKQDU6IvIiLSQQX4uDE8JZThKaFYrTYOlp0m78I0n9VbvmfV5oO4mV3oHXlhUW9sICEBns7utoh0EEr0RUREOgGj0UBMD19ievgy7uYYztado7Sqjq93HSGvuIKd+08AYPF3J+nCgV29owLwcNP/6kVuVPrbLyIi0gl5ursytG8Penb3xmazcfxkrX0nn815x/jnt0dwMRqIC72wd39sEFHdfTBqUa/IDUOJvoiISCdnMBgICfQkJNCT29PDOXfeyv7D1fbEf+mXJSz9sgRvDxN97Hv3BxHg4+bsrovINaREX0REpItxdTGSGBVAYlQAE0fEcaqmgd0HKskrrmT3gUq2FRwHIMzi1bR3f0wQ8RF+mFy1d79IV6JEX0REpIvz9TIzNKk7Q5O6Y7XZOHz8jH0Lz893HGbttkOYXI0kRFzYuz82iNAgT+3dL9LJKdEXERG5gRgNBiJDfIgM8eGuIVHUN5xnz/cn7Yn/B+v3w/r9BPi4XZjiE0if6EC8PUzO7rqIOEiJvoiIyA3MzexCas9gUnsGA3Ci+n8X9e4oLOer3KMYDBDTw9d+Um9sqC8uRu3dL9LRKdEXERERu2A/D0b0C2NEvzDOW62UlJ4mr6Rp7/4VXx/gfzYdwMPNlT5RASTFBpIcHUiwv4ezuy0irVCiLyIiIq1yMRrpGe5Hz3A/xg+P5UxtIwUHT9oP7dqxtxyAkEDPC4t6A0mMDMDNrEW9Ih2BEn0RERG5LN4eJgYmdmNgYjdsNhulFWcvzO2v4MtdpXy+4zAuRgO9wv1Ijm06tCuim7cW9Yo4iRJ9ERERcZjBYCAs2IuwYC/uHBhB47nz7D1UbU/8P/5nER//swhfLzNJ0YH2+f2+XmZnd13khqFEX0RERK6aydWFpAvJ/CR6cvJ0vT3p/664gs27jwEQGeJNckzTaH/PcD9cXbSoV+RaUaIvIiIi7S7Ax41hKT0YltIDq83GwWOnm3bzKa5g7bbv+WTLQdxMLiRG+tun+XQL8NA0H5F2pERfRERErimjwUBMD19ievgy7qZoauvPUXDwpH3Ef1dRBQDBfu4XpvgE0TsqAE93pSkiV0N/g0REROS68nBzpX+8hf7xFgDKTp4lr7hp7/7N+WX8c2cpRoOBuDBfe+If3d0Ho1Gj/SKOUKIvIiIiThUS4ElIuie3p4dz7ryVoiPV5F04qXfplyUs/bIEL3dX+xqA5JggAnzcnN1tkQ5Pib6IiIh0GK4uRhIiA0iIDOCnt8Zx6mwD+ReS/t0llWwrOA5AWLBXU9IfG0h8uD9mk/buF/khJfoiIiLSYfl6mhmS1J0hSd2x2WwcOn6G3QcqySuuZH3OYdZ9cwiTq5H4CH/7oV2hwV5a1CuCkxP9hoYG5s+fz/Llyzl16hSJiYnMmDGDoUOHXrJuWVkZc+bMYdOmTVitVoYMGcKsWbOIiIiw35Odnc2sWbPabOPll1/m7rvvBuDVV1/lb3/7W4t7goOD2bRp0xU8nYiIiLQng8FAZIgPkSE+3DU4ivqG8xQeOmkf7f9w/X4+pGnHn6ToptH+PtGBeHuYnN11EadwaqI/c+ZM1q1bR2ZmJlFRUSxdupRHH32UhQsXkpaW1ma9mpoaMjMzqamp4fHHH8fV1ZWsrCwyMzNZtmwZfn5+AAwcOJCXXnqpRf133nmHPXv2tPoLxezZs3F3d7f/+V//W0RERDoON7MLKXHBpMQFA3CiuvbCTj6V7NhbzlffHcUARPdoWtQ7rH84gZ6uuBi1d7/cGJyW6Ofm5rJq1SpmzZrF1KlTARg/fjxjx45l7ty5LFq0qM267733HgcPHiQ7O5s+ffoAMHz4cMaNG0dWVhbTp08HICIiotkIP0BdXR1//OMfGTJkCBaLpUXbd911F76+vu30lCIiInK9BPt5cGu/MG7tF8Z5q5WSo6fJK65gd0klKzcfYMXXB/Bwc6F3VKB9mk+wv4ezuy1yzTgt0V+zZg0mk4n77rvPXubm5sbEiRN55ZVXOH78ON26dWu17tq1a+nXr589yQeIi4tj6NChrF692p7ot2b9+vXU1NQwbty4Vq/bbDbOnDmDl5fm94mIiHRWLkYjPcP86Bnmx/jhsdTUNXKkso6vdx0mr6SSnL3lAIQEeJAcE0RSbCCJkf64m7V8UboOp73NBQUFxMTE4OXl1aw8JSUFm81GQUFBq4m+1WqlsLCQ+++/v8W1vn37smnTJmpra/HwaP039BUrVuDu7s4dd9zR6vURI0Zw9uxZvLy8GD16NE8++ST+/v5X8IQiIiLSUXi5m7g5NZD4UB9sNhtHK87a5/Z/mVvK5zmHcTEa6BXuZ9/CMyLEG6MG/aQTc1qiX15eTkhISIvyi9Npjh8/3mq9qqoqGhoaWp12Y7FYsNlslJeXExkZ2WrdL7/8klGjRuHt7d3smq+vLw8++CCpqamYTCa2bNnChx9+SH5+PosXL8ZsNl/JY4qIiEgHYzAYCA32IjTYizsHRtB47jx7D1ezu7hpfv+SjcUs2ViMr6fJnvT3iQnEz0u5gHQuTkv06+rqMJlaroJ3c2s6AKO+vr7VehfLW0u8L9atq6trte7atWtpbGxsddrOQw891OzPGRkZ9OrVi9mzZ7Ns2TImTZr0I0/TuqAg70vfdA1YLD5O+dzOSvFynGLmGMXLMYqXYxQvx7QVr9Ae/owYGAVA5ak6vi08Tk7hcXbuLWfz7jIAYkP9SEuw0D+xG72jAzG5dv29+/V+Oaajxctpib67uzuNjY0tyi8m8heT9h+6WN7Q0NBm3bZ2ylmxYgX+/v7ccsstl9XHyZMn8/LLL7N58+YrSvQrKs5gtdocrnc1LBYfystPX9fP7MwUL8cpZo5RvByjeDlG8XKMI/FKiQ4gJToA653xfF92mrwLo/3LNhaxZMN+3EwuJERe2Ls/NoiQAI8ut7ZP75djnBUvo9HQ5uCy0xJ9i8XS6vSc8vKmxTFtLcT19/fHbDbb7/thXYPB0Oq0ntLSUrZv386kSZNa/SahNUajkZCQEKqrqy/rfhEREelajAYD0d19ie7uy9iboqmtP8eegyfJO1DJ7uJKcosqgH0E+7lfmOYTSO+oQDzdtahXnM9pb2FiYiILFy6kpqam2YLcXbt22a+3xmg0Eh8fT15eXotrubm5REVFtboQd+XKldhsNvsBWZejsbGRo0ePkpycfNl1REREpOvycHMlLd5CWvyFNYUnmxb15hVXsiW/jI07SzEaDMSGNe3dnxQTSEx3X4zGrjXaL52D006MyMjIoLGxkcWLF9vLGhoayM7Opn///vaFuqWlpRQVFTWrO3r0aHbu3El+fr69rLi4mC1btpCRkdHq561cuZLQ0FDS09NbvV5ZWdmibMGCBdTX1zN8+HCHn09ERES6vm4BnozsH86vJ6bw6vThPDkljbuGRNJ4zsryL0v4/9/dwfT//JL/XpbHl7tKqTzV+jpCkWvBaSP6qampZGRkMHfuXPsuOUuXLqW0tJQXXnjBft+TTz7Jtm3bKCwstJdNmTKFxYsX89hjjzFt2jRcXFzIysrCYrHYD9/6V3v37qWwsJDHHnuszflzt912G2PGjCE+Ph6z2czWrVtZu3Yt6enpjB07tt2fX0RERLoWVxcjCZEBJEQG8NNb4zh1toH8C1N88koq+WZP05Tl0GAv+4Fd8RH+mE1df1GvOIdTJ5C99NJLzJs3j+XLl1NdXU1CQgKvv/56m6PuF3l7e7Nw4ULmzJnDa6+9htVqZfDgwTz99NMEBAS0uH/FihUAP5qwjxs3jpycHNasWUNjYyNhYWE88cQT/OIXv8DVVfPsRERExDG+nmaG9OnOkD7dsdlsHC6vIa+k6aTe9TmHWffNoaZfDiL8SIoJIjk2kLBgHdgp7cdgs9mu77YwNxDtutPxKV6OU8wco3g5RvFyjOLlmI4Ur/rG8xR+X2VP/I9WnAUgwMeNpOimuf1JMYF4e1zeBiLXQkeKV2egXXdEREREBDeTCylxQaTEBQFQUV3H7gOV5BVXkLO3nK++O4oBiO7h0zTaHxNIbKgvri5OW14pnZASfREREREnC/Jz55bUUG5JDeW81cqBo6ebdvMpqWDV5gOs/PoAHm4uJEYGkBzblPhb/FvuMijyr5Toi4iIiHQgLkYjcWF+xIX5cc+wGGrqGik4cJK8kkp2l1Tw7b4TAIQEeFzYuz+IxCh/3M1K66Q5vREiIiIiHZiXu4kBid0YkNgNm83Gscqz9pN6v8o9yvqcI7gYDfQK97Mn/hEh3hi1qPeGp0RfREREpJMwGAz0CPKiR5AXdwyMoPGclX2Hq+yHdi3ZWMySjcX4eproc2ELz6ToQPy83ZzddXECJfoiIiIinZTJ1Uif6ED6RAcy6TaoOlPP7pJKdl88rXd3GQAR3bzte/f3DPfH5KpFvTcCJfoiIiIiXYS/txs39+3BzX17YLXZ+L7stD3pX/fNIVZv/R6zyUhiZMCFaT6BdA/01N79XZQSfREREZEuyGgwEN3dl+juvvxkaDS19efY8/3FRb2V5BZVABDk605ybNMUnz7RAXi6O2/vfmlfSvRFREREbgAebq6k9bKQ1ssCwPGqWnYXV5BXUsnW/DI27izFaDAQG+pLckwgN/cPJ8DdFaNRo/2dlRJ9ERERkRtQN38PuvUP57b+4Zw7b6W49BR5JRXkFVey/KsSln1Vgpe7K72jA+3z+wN93Z3dbXGAEn0RERGRG5yri5H4CH/iI/yZcEscp882cLiylq93HSGvpJLte44DEBrsRVJ0IMmxgcRH+ONmcnFyz+XHKNEXERERkWZ8PM3cEhVE73A/bDYbR8pr7Ad2bfj2CJ9uP4Sri5GECD+SYppO6g2zeGlRbwejRF9ERERE2mQwGAjv5k14N28yBkdS33ievYeqLhzaVcFHG/bz0Qbw9zaTFBPY9E90ID6eZmd3/YanRF9ERERELpubyYW+sUH0jQ0CelF5qq7pwK6SSnbuO8Gm745hAKK6+5Ac23RSb2yoL64u2rv/elOiLyIiIiJXLNDXnVtSQ7klNRSr1UbJsVPsLm5K/FdtPsjKrw/ibnahd1RA00m9sUF08/dwdrdvCEr0RURERKRdGI0G4kL9iAv14+5hMZytayT/wEn7/P5v950AoFuAh/3ArsTIADzclJJeC4qqiIiIiFwTnu4mBiR2Y0BiN2w2G8cqz9oP7Nr03VE25BzBxWigZ5hf06FdMYFEhvhg1KLedqFEX0RERESuOYPBQI8gL3oEeXHHgAgaz1nZf7jKPr9/ycZilmwsxsfTRFJ0oH1hr7+3m7O73mkp0RcRERGR687kaqR3dCC9owO57zaoPlPP7gOV9hH/LfllAIRbvC8s6g2kV7g/Jlct6r1cSvRFRERExOn8vN24KbkHNyX3wGqzcajsDHklFewuqeTTbw6xZuv3mE1GEiMD7Id2dQ/01N79P0KJvoiIiIh0KEaDgajuPkR19+EnQ6OprT9H4fdV5JVUkFdSSW5RBXwOQb5u9gO7ekcH4OVucnbXOxQl+iIiIiLSoXm4udKvVzD9egUDcLyqlt0lleQVV7CtoIwvdpViMEBsqC/JFxL/mB6+GI039mi/En0RERER6VS6+XvQLS2M29LCOHfeSnHpKfsWnv/zVQnLvyrB082VPtEBJMc2Jf6Bvu7O7vZ1p0RfRERERDotVxcj8RH+xEf4M+GWWE6fbSD/wMmmEf+SCrYXlgPQI8jzwt79QSRE+uNmcnFyz689JfoiIiIi0mX4eJoZ3CeEwX1CsNlsHDlRQ15x02j/P78t5bPthy/8cuBHckwQSTGBhFu8uuSiXqcm+g0NDcyfP5/ly5dz6tQpEhMTmTFjBkOHDr1k3bKyMubMmcOmTZuwWq0MGTKEWbNmERERYb8nOzubWbNmtdnGyy+/zN133+1QmyIiIiLSORgMBsIt3oRbvMkYHElD43n2Hvrfvfs/2rAfNoCft5nk6ECSYgPpEx2Ir6fZ2V1vFwabzWZz1of/9re/Zd26dWRmZhIVFcXSpUvJy8tj4cKFpKWltVmvpqaGCRMmUFNTw9SpU3F1dSUrKwuDwcCyZcvw8/MD4NChQ+Tk5LSo/84777Bnzx42btyIxWJxqE1HVFScwWq9vuG1WHwoLz99XT+zM1O8HKeYOUbxcozi5RjFyzGKl2NuhHhVnqq7MMWnkvwDldTUncMARHb3ITmmae/+uDA/XF0uvXe/s+JlNBoICvJu9ZrTRvRzc3NZtWoVs2bNYurUqQCMHz+esWPHMnfuXBYtWtRm3ffee4+DBw+SnZ1Nnz59ABg+fDjjxo0jKyuL6dOnAxAREdFiNL6uro4//vGPDBkyxJ7kO9KmiIiIiHQNgb7uDE8NZXhqKFarjZJjp+yJ/+ot37Nq80HczC70jgywH9rVLcCzWRubdx8je2MRlafqCfR1Y8KtcQxN6u6kJ2rOaYn+mjVrMJlM3HffffYyNzc3Jk6cyCuvvMLx48fp1q1bq3XXrl1Lv3797Ak5QFxcHEOHDmX16tU/mpSvX7+empoaxo0b125tioiIiEjnZjQaiAv1Iy7Uj7tvjuFsXSMFB0/aE/+d+08ATTv+JMUGkhwdyKmzDbz/2T4azlkBqDhVzzur9wB0iGTfaYl+QUEBMTExeHl5NStPSUnBZrNRUFDQaqJvtVopLCzk/vvvb3Gtb9++bNq0idraWjw8PFr93BUrVuDu7s4dd9zRbm2KiIiISNfi6W4iPaEb6QndsNlslJ2sJa+46cCuTd8dZUPOkVbrNZyzkr2x6MZO9MvLywkJCWlRfnE6zfHjx1utV1VVRUNDQ7NpN/9a12azUV5eTmRkZKt1v/zyS0aNGoW3t3ez8ittU0RERES6NoPBQPdAT7oHejJqQASN56zsP1LNy+9/2+r9Fafqr3MPW+e0RL+urg6TqeUxxW5ubgDU17ceoIvlZnPL1dAX69bV1bVad+3atTQ2NraYtnM1bf6YthZGXGsWi49TPrezUrwcp5g5RvFyjOLlGMXLMYqXYxSvtoX28CNrzR7KT9a2uGYJ8OgQsXNaou/u7k5jY2OL8otJ98UE+4culjc0NLRZ19299ZPPVqxYgb+/P7fccku7tfljtOtOx6d4OU4xc4zi5RjFyzGKl2MUL8coXpc2flgM76zeY5+jD2B2NTJ+WMx1i12H3HXHYrG0Oj2nvLzp9LK2FuL6+/tjNpvt9/2wrsFgaHUKTmlpKdu3b2fSpEktvkm40jZFRERE5MZ1cR6+dt35gcTERBYuXEhNTU2zBbm7du2yX2+N0WgkPj6evLy8Ftdyc3OJiopqddHsypUrsdlszQ7Iuto2RUREROTGNjSpO0OTunfIb0Auvfv/NZKRkUFjYyOLFy+2lzU0NJCdnU3//v3tC3VLS0spKipqVnf06NHs3LmT/Px8e1lxcTFbtmwhIyOj1c9buXIloaGhpKent3r9StoUEREREemonDain5qaSkZGBnPnzrXvaLN06VJKS0t54YUX7Pc9+eSTbNu2jcLCQnvZlClTWLx4MY899hjTpk3DxcWFrKwsLBaL/fCtf7V3714KCwt57LHHMBgMrfbH0TZFRERERDoypyX6AC+99BLz5s1j+fLlVFdXk5CQwOuvv97mqPtF3t7eLFy4kDlz5vDaa69htVoZPHgwTz/9NAEBAS3uX7FiBQBjx45ttzZFRERERDoyg81mu77bwtxAtOtOx6d4OU4xc4zi5RjFyzGKl2MUL8coXo5xVrx+bNcdp83RFxERERGRa0eJvoiIiIhIF6REX0RERESkC1KiLyIiIiLSBSnRFxERERHpgpy6vWZXZzS2vmd/V/3czkrxcpxi5hjFyzGKl2MUL8coXo5RvBzjjHj92Gdqe00RERERkS5IU3dERERERLogJfoiIiIiIl2QEn0RERERkS5Iib6IiIiISBekRF9EREREpAtSoi8iIiIi0gUp0RcRERER6YKU6IuIiIiIdEFK9EVEREREuiAl+iIiIiIiXZCrszsgzTU0NDB//nyWL1/OqVOnSExMZMaMGQwdOvSSdcvKypgzZw6bNm3CarUyZMgQZs2aRURERIt7Fy9ezFtvvcXhw4cJDQ0lMzOTBx544Fo80jV1pfFat24dn3zyCbm5uVRUVNCjRw9uu+02nnjiCXx8fJrdm5CQ0Gobzz33HJMnT263Z7kerjRer776Kn/7299alAcHB7Np06YW5Tf6+zVy5EiOHDnS6rWoqCjWrVtn/3NXer+OHz/Ou+++y65du8jLy+Ps2bO8++67DB48+LLqFxUVMWfOHHJycjCZTNx22208+eSTBAYGNrvParWyYMEC3n//fcrLy4mOjuaXv/wlY8aMuRaPdc1cabysVitLly7l008/paCggOrqasLDwxk7diwPP/wwZrPZfu/hw4e5/fbbW23njTfe4JZbbmnXZ7qWrub9mjlzJkuXLm1RnpqaykcffdSs7EZ/v6Dtn0sAN910E2+//TbQtd6v3Nxcli5dytatWyktLcXf35+0tDR+85vfEBUVdcn6HTUHU6LfwcycOZN169aRmZlJVFQUS5cu5dFHH2XhwoWkpaW1Wa+mpobMzExqamp4/PHHcXV1JSsri8zMTJYtW4afn5/93g8++IBnn32WjIwMpk2bxvbt25k9ezb19fU8/PDD1+Mx282VxusPf/gD3bp145577iE0NJTCwkIWLlzIl19+yZIlS3Bzc2t2/7Bhw7j77rublaWmpl6TZ7qWrjReF82ePRt3d3f7n//1vy/S+wVPPfUUNTU1zcpKS0uZN28eN998c4v7u8r7VVJSwhtvvEFUVBQJCQl8++23l1332LFjPPDAA/j6+jJjxgzOnj3LW2+9xd69e/noo48wmUz2e1955RVef/117r//fpKTk/n888+ZMWMGRqORjIyMa/Fo18SVxqu2tpannnqKfv368bOf/YygoCC+/fZb5s+fz5YtW8jKympR5+6772bYsGHNyhITE9vjMa6bq3m/ADw8PPjjH//YrOyHv0SC3i+Al156qUVZXl4e7777bqs/w7rC+/Xmm2+Sk5NDRkYGCQkJlJeXs2jRIsaPH8/HH39MXFxcm3U7dA5mkw5j165dtvj4eNvbb79tL6urq7ONGjXKNmXKlB+t+/rrr9sSEhJsu3fvtpft37/f1rt3b9u8efPsZbW1tbZBgwbZfvnLXzar/x//8R+2tLQ026lTp9rnYa6Dq4nXli1bWpQtXbrUFh8fb1uyZEmz8vj4eNuf/vSndumzM11NvP7zP//TFh8fb6uurv7R+/R+te2//uu/bPHx8bYdO3Y0K+8q75fNZrOdPn3aVllZabPZbLZPP/3UFh8f3+rftdY8++yztn79+tmOHTtmL3eeap0AABECSURBVNu0aZMtPj7etnjxYnvZsWPHbElJSc1iZrVabVOmTLHddttttvPnz7fT01x7Vxqv+vr6Fu+RzWazvfrqqy3aOHToUIv3uLO6mvfrySeftKWnp1/yPr1fbXvqqadsCQkJtqNHj9rLutL7tWPHDlt9fX2zspKSEltycrLtySef/NG6HTkH0xz9DmTNmjWYTCbuu+8+e5mbmxsTJ05kx44dHD9+vM26a9eupV+/fvTp08deFhcXx9ChQ1m9erW9bOvWrVRVVTFlypRm9R944AFqamr44osv2vGJrq2riVdrX12OGjUKaJo+0Jq6ujrq6+uvstfOczXxushms3HmzBlsNlur1/V+tW3lypWEh4fTv3//Vq939vcLwNvbm4CAgCuqu27dOkaOHElISIi97KabbiI6OrrZz7DPPvuMxsbGZu+YwWBg8uTJHDlyhNzc3Ct/gOvsSuNlNptbfY/uuOMOoO2fYWfPnqWhocHhz+sorub9uuj8+fOcOXOmzet6v1rX0NDAunXrGDhwIN27d2/1ns7+fvXv37/ZtDeA6OhoevXq1ebfqYs6cg6mRL8DKSgoICYmBi8vr2blKSkp2Gw2CgoKWq1ntVopLCwkOTm5xbW+ffty4MABamtrAcjPzwdocW9SUhJGo9F+vTO40ni15cSJEwCt/mD8+OOP6devHykpKYwbN45PP/30yjvuJO0RrxEjRpCenk56ejqzZs2iqqqq2XW9X63Lz8+nqKiIsWPHtnq9K7xfV6OsrIyKiopWf4alpKQ0i3VBQQHe3t7ExMS0uA/oVO9Ye/uxn2Hz588nLS2NlJQU7r//fr755pvr3T2nq6mpsf/8Gjx4MC+88EKLX671frVu48aNnDp1qsUUw4u66vtls9k4ceLEj/7C1NFzMM3R70DKy8ubjWZdZLFYANocQayqqqKhocF+3w/r2mw2ysvLiYyMpLy8HLPZjL+/f7P7LpY5OkrpTFcar7a88cYbuLi4cOeddzYrT0tLY8yYMYSHh3P06FHeffddfvWrX/GXv/ylzcStI7qaePn6+vLggw+SmpqKyWRiy5YtfPjhh+Tn57N48WL7KIjer9atWLECoNX/SXaV9+tqXIxlWz/DKioqOH/+PC4uLpSXlxMcHNzqff/a1o3ozTffxMfHp9lcaaPRyLBhw7jjjjvo1q0bBw8eZMGCBUybNo2srCwGDBjgxB5fPxaLhZ///Of07t0bq9XKhg0byMrKoqioiDff/H/t3XtQVOUbB/AvSxBiEQMCAYJKKYgyYKQChYKkEuAQM02kXAwBUcK8ZFlDTRPVT2agUSMbSXCaZlJMs7joaCo4Jks4UG4ZkQko7CCXEbkNsm4uvz+cPcNxd3XlorvL9/Pfec9zbq8Px2f2vOc9BUIc80u70tJSWFpaYvny5aJ2U8+vkpIStLe3Y/PmzTpjDL0GY6FvQAYHB0UvnKmpXwzV9Vhf3X73I6fh2w4ODt7zGOpYYxo6MNL+0qa0tBSHDx9GWloa3N3dReuKiopEyzExMYiKikJOTg4iIyNhZmY2grN/+EbTX6tXrxYth4eHY+bMmcjKysJPP/2E11577Z7HUB9nIuaXSqXC0aNH4e3trfVlLlPJr9HQ9x42efJkDA4O3jPOmHJsLO3ZswdSqRRZWVmimcNcXFxQWFgoio2IiEBkZCRyc3M18s9Uvf3226LlqKgoODk5obCwEJWVlcILpswvTf39/Thz5gwWL14MGxsb0TpTzq+GhgZkZWXB398f0dHROuMMvQbj0B0DYmVlBaVSqdGu/oe/eyYYNXW7trFx6m3Vs6NYWVnpHEOnUCh0HsMQjbS/7lZTU4PMzEyEhIRg48aN9423trbG66+/jra2NjQ2Nj7YST9CY9VfaitXrsSkSZNQVVUlOgbzS+z8+fNob2/HihUr9Io31vwajbG4h400j03BsWPHsHPnTsTGxiI2Nva+8U5OToiMjIRMJhOGFExE6hlO9LmHTeT8OnHiBBQKhd73MFPIr87OTqSlpeGpp57Crl27IJHoLpcNvQZjoW9AHBwctD626ezsBAA4Ojpq3c7W1haWlpZC3N3bmpmZCY+UHBwcoFQqNcZW37p1C93d3TqPYYhG2l/D1dfXY/369fD09MSOHTtgbm6u17GdnZ0BAD09PQ9wxo/WWPTXcBKJBE5OTqI+YH5pKi0thUQiQWRkpN7HNsb8Gg11X+q6h9nb2wt/mw4ODsJY9Lvjhu9roqisrMS7776L0NBQfPTRR3pv5+zsDJVKhd7e3nE8O8M2ZcoUWFhYaNzDmF9ipaWlePLJJxEaGqr3NsacX319fUhNTUVfXx8KCgq0DskZztBrMBb6BsTLywtNTU0a82/LZDJhvTYSiQSzZs3CxYsXNdb98ccfmDZtGiZNmgQAmD17NgBoxF68eBEqlUpYbwxG2l9qzc3NSElJgZ2dHfLz82Ftba33sVtaWgBon4PZUI22v+6mVCpx7do10UtKzC8x9UwVCxYs0DreXxdjzK/RcHJygp2dnc572PC8mT17Nvr7+9HU1CSKU/+7GFOOjZZMJkNGRgZ8fHwe6IcK4E6OmZubi+b3nmja2tqgVCpFf2fML7GOjg5UV1dj2bJlWoem6GKs+aVQKLBu3TpcuXIF+fn58PDwuO82hl6DsdA3IOHh4VAqlTh06JDQduvWLRw5cgTPPfecUCi0trZqTPW0fPlyXLhwQfTGdmNjI3799VfRBz4CAgJga2uL/fv3i7Y/cOAArK2tjeordqPpr87OTqxZswZmZmYoLCzUWVB1dXVptN24cQP79+/H1KlTMX369LG7oHE2mv7S1g+FhYVQKBQIDg4W2phfYuqZKnQ98jal/HoQzc3NaG5uFrUtW7YM5eXlaG9vF9qqqqpw5coV0T0sLCwMFhYWohwbGhpCUVERXFxcjPJDY/ejrb8aGhqwdu1auLq6Ys+ePVo/Xgdoz7GrV6/i6NGjeP7553VuZ8zu7i+FQqF1Ss2vvvoKAEQvLzO/xI4dOwaVSvVA9zBjza/bt29j06ZNuHDhAnbt2gU/Pz+tccZWg/FlXAPi6+uL8PBw5ObmCm9o//jjj2htbcX27duFuG3btuH8+fP4559/hLZVq1bh0KFDWLt2LZKSkmBubo5vvvkGDg4OeOONN4Q4KysrvPXWW8jKysLGjRvx4osvoqamBiUlJdi6davGizaGbDT9lZKSgpaWFqSkpKC2tha1tbXCOnd3d+Grp9999x1Onz6NkJAQuLi4oL29HQcPHkRXVxd279798C52DIymv0JDQxEREYFZs2bB0tIS1dXVOHHiBPz9/UUzwzC/xHTNVKFmSvmlpi6e1P8RFhcXo7a2FjY2NoiPjwcA4Z5UXl4ubLdu3TocP34ciYmJiI+Px8DAAAoLC+Hl5SV6Ee7pp59GYmIi9u3bB4VCAR8fH5w6dQo1NTXYsWPHPcfSGqKR9Fd/fz+Sk5PR29uL5ORknDlzRrRPT09P4YlTTk4OWlpaEBAQAEdHRzQ3NwsvSG7btm28L2/MjaS/Ojs7hZfcPTw8hFl3qqqqEBERgfnz5wv7Z36JlZSUwNHRUeu3ZwDTyq/s7GyUl5cjNDQU3d3dKC4uFtZNnjxZ+NaOsdVgZkO6vnxDj4RCocDOnTtRWlqKnp4eeHp6YsuWLQgKChJiEhIStBYWbW1t+N///ofKykqoVCosXLgQmZmZcHNz0zjO999/j3379kEul8PZ2RkJCQlITEwc9+sbayPtL09PT537jImJQXZ2NgDg3LlzKCwsxKVLl9DT0wNra2v4+fkhLS0N/v7+43dh42Sk/fXBBx/gt99+w7Vr16BUKuHq6oqIiAikpaVp/cVmoucXcKcYCwoKwuLFi5GXl6d1/6aWX4Duvy1XV1ehkFiyZAkAzcLi33//RXZ2Nmpra2FhYYGQkBC8//77Gk/cVCoV9u7di4MHD6KjowMzZsxAWlqaUU5HOpL+ksvlCAsL07nPjIwMbNiwAcCdD7UVFRXh8uXL6Ovrg42NDRYsWICMjAzMnDlzLC/loRhJf/X29uKTTz6BTCZDR0cHVCoVpk+fjpiYGCQmJmoMeZro+aXW2NiIl19+GUlJSXjvvfe07seU8kt9L9dmeH8ZWw3GQp+IiIiIyAQZ1zMoIiIiIiLSCwt9IiIiIiITxEKfiIiIiMgEsdAnIiIiIjJBLPSJiIiIiEwQC30iIiIiIhPEQp+IiIiIyASx0CciIpOSkJAgfASIiGgie+xRnwARERm+6urqe3650dzcHHV1dQ/xjIiI6H5Y6BMRkd6ioqKwaNEijXaJhA+IiYgMDQt9IiLSm7e3N6Kjox/1aRARkR74EwwREY0ZuVwOT09P5OXloaysDCtWrICPjw9CQkKQl5eH//77T2Ob+vp6vPnmm1i4cCF8fHwQERGBvXv34vbt2xqxnZ2d+PTTTxEWFoa5c+ciMDAQSUlJqKys1Ihtb2/Hli1bMH/+fPj6+iI5ORlNTU3jct1ERIaIv+gTEZHebt68ia6uLo12S0tLPPHEE8JyeXk5WlpaEBcXhylTpqC8vBxffvklWltbsX37diHuzz//REJCAh577DEhtqKiArm5uaivr8fnn38uxMrlcqxcuRLXr19HdHQ05s6di5s3b0Imk0EqleKFF14QYgcGBhAfHw9fX19s3rwZcrkc3377LdLT01FWVgZzc/Nx6iEiIsPBQp+IiPSWl5eHvLw8jfaQkBDk5+cLy/X19Th8+DDmzJkDAIiPj0dGRgaOHDmC2NhY+Pn5AQA+++wz3Lp1C0VFRfDy8hJiN23ahLKyMrz66qsIDAwEAHz88cfo6OhAQUEBgoODRcdXqVSi5Rs3biA5ORmpqalCm52dHXJyciCVSjW2JyIyRSz0iYhIb7GxsQgPD9dot7OzEy0HBQUJRT4AmJmZISUlBadOncLJkyfh5+eH69ev4/fff8fSpUuFIl8du379ehw/fhwnT55EYGAguru78csvvyA4OFhrkX73y8ASiURjlqCAgAAAwNWrV1noE9GEwEKfiIj0Nm3aNAQFBd037plnntFoe/bZZwEALS0tAO4MxRnePpyHhwckEokQ29zcjKGhIXh7e+t1no6Ojnj88cdFbba2tgCA7u5uvfZBRGTs+DIuERGZnHuNwR8aGnqIZ0JE9Oiw0CciojHX0NCg0Xb58mUAgJubGwBg6tSpovbhGhsboVKphFh3d3eYmZnh77//Hq9TJiIyOSz0iYhozEmlUvz111/C8tDQEAoKCgAAL730EgDA3t4e8+bNQ0VFBS5duiSK/frrrwEAS5cuBXBn2M2iRYtw9uxZSKVSjePxV3oiIk0co09ERHqrq6tDcXGx1nXqAh4AvLy8sHr1asTFxcHBwQGnT5+GVCpFdHQ05s2bJ8RlZmYiISEBcXFxWLVqFRwcHFBRUYFz584hKipKmHEHAD788EPU1dUhNTUVr7zyCubMmQOFQgGZTAZXV1e8884743fhRERGiIU+ERHpraysDGVlZVrX/fzzz8LY+CVLlmDGjBnIz89HU1MT7O3tkZ6ejvT0dNE2Pj4+KCoqwhdffIEDBw5gYGAAbm5u2Lp1K9asWSOKdXNzww8//IDdu3fj7NmzKC4uho2NDby8vBAbGzs+F0xEZMTMhvi8k4iIxohcLkdYWBgyMjKwYcOGR306REQTGsfoExERERGZIBb6REREREQmiIU+EREREZEJ4hh9IiIiIiITxF/0iYiIiIhMEAt9IiIiIiITxEKfiIiIiMgEsdAnIiIiIjJBLPSJiIiIiEwQC30iIiIiIhP0f5/ILMymhHM8AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-ZqpKH4RgfQ"
      },
      "source": [
        "test_sentence = \"\"\"\n",
        "Mr. Trump’s tweets began just moments after a Fox News report by Mike Tobin, a \n",
        "reporter for the network, about protests in Minnesota and elsewhere. \n",
        "\"\"\"\n"
      ],
      "execution_count": 137,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "irtvfJHiRgqH"
      },
      "source": [
        "tokenized_sentence = tokenizer.encode(test_sentence)\n",
        "input_ids = torch.tensor([tokenized_sentence]).cuda()\n"
      ],
      "execution_count": 138,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Clxo8sVzRg00"
      },
      "source": [
        "with torch.no_grad():\n",
        "    output = model(input_ids)\n",
        "label_indices = np.argmax(output[0].to('cpu').numpy(), axis=2)\n"
      ],
      "execution_count": 139,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IpaFj-zvtVCE"
      },
      "source": [
        "# join bpe split tokens\n",
        "tokens = tokenizer.convert_ids_to_tokens(input_ids.to('cpu').numpy()[0])\n",
        "new_tokens, new_labels = [], []\n",
        "for token, label_idx in zip(tokens, label_indices[0]):\n",
        "    if token.startswith(\"##\"):\n",
        "        new_tokens[-1] = new_tokens[-1] + token[2:]\n",
        "    else:\n",
        "        new_labels.append(tag_values[label_idx])\n",
        "        new_tokens.append(token)\n"
      ],
      "execution_count": 140,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MzZ6Sf_rtXK4",
        "outputId": "342cc9be-0a0f-4cf5-88b8-ab92822a7a96"
      },
      "source": [
        "for token, label in zip(new_tokens, new_labels):\n",
        "    print(\"{}\\t{}\".format(label, token))\n"
      ],
      "execution_count": 141,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "O\t[CLS]\n",
            "B-per\tMr\n",
            "B-per\t.\n",
            "I-per\tTrump\n",
            "O\t’\n",
            "O\ts\n",
            "O\ttweets\n",
            "O\tbegan\n",
            "O\tjust\n",
            "O\tmoments\n",
            "O\tafter\n",
            "O\ta\n",
            "B-org\tFox\n",
            "I-org\tNews\n",
            "O\treport\n",
            "O\tby\n",
            "B-per\tMike\n",
            "I-per\tTobin\n",
            "O\t,\n",
            "O\ta\n",
            "O\treporter\n",
            "O\tfor\n",
            "O\tthe\n",
            "O\tnetwork\n",
            "O\t,\n",
            "O\tabout\n",
            "O\tprotests\n",
            "O\tin\n",
            "B-geo\tMinnesota\n",
            "O\tand\n",
            "O\telsewhere\n",
            "O\t.\n",
            "O\t[SEP]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}