ULMFiT LM Training example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from fastai import *\n",
    "from fastai.text import * "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('data/lm/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data format used in this notebook has changed, to comply with the new fastai API.\n",
    "\n",
    "The new format is a CSV file with two columns:\n",
    "\n",
    "- First column: text content to train the model. Here we have 100M words for the training data + 22M words for the validation data.\n",
    "- Second column: Boolean representing if the data is for training or validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(PATH/'train.csv', header=None, names=['label', 'content'])\n",
    "valid = pd.read_csv(PATH/'valid.csv', header=None, names=['label', 'content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['is_valid'] = False\n",
    "valid['is_valid'] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = train[['content', 'is_valid']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = valid[['content', 'is_valid']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(                                                 content  is_valid\n",
       " 0      Guerres civiles colombiennes\\n\\nLes guerres ci...     False\n",
       " 1      Présidence de Nicolas Sarkozy\\n\\nLa présidence...     False\n",
       " 2      Planification en république populaire de Chine...     False\n",
       " 3      Worst Case Execution Time\\n\\nLe WCET ou Worst ...     False\n",
       " 4      Église Saint-Denis de Mogneville\\n\\nL'église S...     False\n",
       " 5      Église Saint-Denis de Remy\\n\\nL'église Saint-D...     False\n",
       " 6      Église Sainte-Marie-Madeleine du Bellay-en-Vex...     False\n",
       " 7      Église Sainte-Maxence de Pont-Sainte-Maxence\\n...     False\n",
       " 8      Église Saint-Didier de Villiers-le-Bel\\n\\nL'ég...     False\n",
       " 9      Église Saint-Étienne de Marly-la-Ville\\n\\nL'ég...     False\n",
       " 10     Église Sainte-Madeleine de Trie-Château\\n\\nL'é...     False\n",
       " 11     Église Saint-Germain d'Andrésy\\n\\nL'église Sai...     False\n",
       " 12     Église Saint-Germain-de-Paris de Cléry-en-Vexi...     False\n",
       " 13     Église Saint-Gervais de Pontpoint\\n\\nL'église ...     False\n",
       " 14     Église Saint-Gervais-et-Saint-Protais de Bessa...     False\n",
       " 15     Linky\\n\\nLinky est le nom du compteur électriq...     False\n",
       " 16     Person of Interest\\n\\nEn Belgique, la série es...     False\n",
       " 17     Pierres noires : Les Classes moyennes du Salut...     False\n",
       " 18     SS Edmund Fitzgerald\\n\\nSS \"Edmund Fitzgerald\"...     False\n",
       " 19     Nicholas Nickleby\\n\\nCommencé de concert avec ...     False\n",
       " 20     Église Saint-Jacques de Compiègne\\n\\nL'église ...     False\n",
       " 21     Église Saint-Josse de Parnes\\n\\nL'église Saint...     False\n",
       " 22     Église Saint-Justin de Louvres\\n\\nL'église Sai...     False\n",
       " 23     Église Saint-Leu-Saint-Gilles de Chennevières-...     False\n",
       " 24     Église Saint-Lucien d'Avernes\\n\\nL'église Sain...     False\n",
       " 25     Église Saint-Martin de Cormeilles-en-Parisis\\n...     False\n",
       " 26     Église Saint-Martin de Groslay\\n\\nL'église Sai...     False\n",
       " 27     Église Saint-Martin de L'Isle-Adam\\n\\nL'église...     False\n",
       " 28     Église Saint-Martin de Triel-sur-Seine\\n\\nL'ég...     False\n",
       " 29     Personnages de Black Butler\\n\\nCette page prés...     False\n",
       " ...                                                  ...       ...\n",
       " 14183  Le Moyen de parvenir\\n\\nLe Moyen de parvenir e...     False\n",
       " 14184  Silkworm (missile)\\n\\nLes séries Shang Yo ou S...     False\n",
       " 14185  Front républicain (Ve République)\\n\\nSous la C...     False\n",
       " 14186  Mesure à cinq temps\\n\\nLa mesure à cinq temps ...     False\n",
       " 14187  Épodes\\n\\nLes Épodes (en latin : ' ou ') sont ...     False\n",
       " 14188  Tour Saint-Rieul\\n\\nLa tour Saint-Rieul est un...     False\n",
       " 14189  Barrage de castors\\n\\nUn barrage de castors es...     False\n",
       " 14190  Histoire de la préfecture de Kumamoto\\n\\nLhist...     False\n",
       " 14191  Notes américaines\\n\\nNotes américaines, en ang...     False\n",
       " 14192  Histoire du Bayern Munich\\n\\nL'histoire du Bay...     False\n",
       " 14193  Histoire de la marine française sous Louis XV ...     False\n",
       " 14194  Fédération des jeunesses communistes de France...     False\n",
       " 14195  Alexander Zverev\\n\\nAlexander Zverev, né le à ...     False\n",
       " 14196  Grande mosaïque de Lillebonne\\n\\nLa grande mos...     False\n",
       " 14197  Faisceaux siciliens\\n\\nLes faisceaux siciliens...     False\n",
       " 14198  Équitation en France\\n\\nL‘équitation est le tr...     False\n",
       " 14199  Leopoldo López\\n\\nLeopoldo López Mendoza, né à...     False\n",
       " 14200  Affaire des caisses noires de la CDU\\n\\nL’affa...     False\n",
       " 14201  Falun Gong hors de Chine continentale\\n\\nLe Fa...     False\n",
       " 14202  Conservateur des hypothèques\\n\\nLe conservateu...     False\n",
       " 14203  Viticulture en Égypte\\n\\nLa viticulture en Égy...     False\n",
       " 14204  Personnages du Visiteur du futur\\n\\nCet articl...     False\n",
       " 14205  Traversée des Ardennes\\n\\nLa traversée des Ard...     False\n",
       " 14206  Bronzes chinois\\n\\nLes bronzes chinois sont un...     False\n",
       " 14207  Littérature néerlandaise de la première moitié...     False\n",
       " 14208  Attaque du Prince of Wales et du Repulse\\n\\nL'...     False\n",
       " 14209  Statuette de Ganymède\\n\\nLa statuette de Ganym...     False\n",
       " 14210  Euromaïdan\\n\\nEuromaïdan (en , ' ; en , ') est...     False\n",
       " 14211  Conquête romaine de l'Étrurie\\n\\nLa conquête r...     False\n",
       " 14212  Bataille des Deux Morins\\n\\nLa bataille des De...     False\n",
       " \n",
       " [14213 rows x 2 columns],\n",
       "                                                 content  is_valid\n",
       " 0     Intouchables (film)\\n\\nIntouchables est un fil...      True\n",
       " 1     Classe Rivadavia\\n\\nLa classe Rivadavia était ...      True\n",
       " 2     Anders Behring Breivik\\n\\nFjotolf Hansen, anci...      True\n",
       " 3     Église Saint-Denis de Sérifontaine\\n\\nL'église...      True\n",
       " 4     Église Saint-Germain-de-Paris d'Hardricourt\\n\\...      True\n",
       " 5     Église Saint-Laurent de Beaumont-sur-Oise\\n\\nL...      True\n",
       " 6     Église Saint-Léger de Jagny-sous-Bois\\n\\nL'égl...      True\n",
       " 7     Église Saint-Martin d'Attainville\\n\\nL'église ...      True\n",
       " 8     Église Saint-Martin de Rousseloy\\n\\nL'église S...      True\n",
       " 9     Règle d'or budgétaire\\n\\nLa règle d'or budgéta...      True\n",
       " 10    Ghost (groupe)\\n\\nGhost est un groupe de heavy...      True\n",
       " 11    Configuration d'aile\\n\\nDepuis les débuts de l...      True\n",
       " 12    Fort de Buenos Aires\\n\\nLe fort de Buenos Aire...      True\n",
       " 13    Rapport Palmer\\n\\nLe rapport Palmer est le rés...      True\n",
       " 14    Le Village indien\\n\\nLe Village indien (\"\") es...      True\n",
       " 15    Dubaï (ville)\\n\\nDubaï ou Doubaï est la premiè...      True\n",
       " 16    Kinot\\n\\nLes kinot (hébreu : קינות \"qinot\" « c...      True\n",
       " 17    Société interculturelle\\n\\nLa société intercul...      True\n",
       " 18    Remparts de Senlis\\n\\nCet article présente l'e...      True\n",
       " 19    Insurrection de Palerme\\n\\nL’Insurrection de P...      True\n",
       " 20    Val-d'Isère\\n\\nVal-d'Isère, également écrite V...      True\n",
       " 21    Dragonica\\n\\nDragonica est un MMORPG 3D full s...      True\n",
       " 22    Spam\\n\\nLe spam, courriel indésirable ou pourr...      True\n",
       " 23    Special Night Squads\\n\\nLes (sans dénomination...      True\n",
       " 24    Yingluck Shinawatra\\n\\nYingluck Shinawatra (en...      True\n",
       " 25    Couvent des Feuillants\\n\\nLe monastère royal d...      True\n",
       " 26    Affaire Caylee Anthony\\n\\nL'affaire Caylee Ant...      True\n",
       " 27    LK (véhicule spatial)\\n\\nLe LK (\"russe \", \"Lou...      True\n",
       " 28    Inazuma Eleven GO\\n\\nLa première saison de l'a...      True\n",
       " 29    Puits Arthur-de-Buyer\\n\\nLe puits Arthur-de-Bu...      True\n",
       " ...                                                 ...       ...\n",
       " 6175  Mini (2001-2006)\\n\\nLa Mini est une automobile...      True\n",
       " 6176  Méthoprène\\n\\nLe méthoprène ou (E, E)-(RS)-11-...      True\n",
       " 6177  Currie Cup (rugby à XV)\\n\\nLa Currie Cup, égal...      True\n",
       " 6178  Tuerie de Chevaline\\n\\nLa tuerie de Chevaline ...      True\n",
       " 6179  ReDRagon\\n\\nLes reDRagon sont une équipe de ca...      True\n",
       " 6180  Histoire de l'Ipswich Town Football Club\\n\\nIp...      True\n",
       " 6181  Heidegger et le problème de l'espace\\n\\nDans s...      True\n",
       " 6182  Système attentionnel superviseur\\n\\nLes foncti...      True\n",
       " 6183  Grand Prix automobile d'Espagne 2014\\n\\nLe Gra...      True\n",
       " 6184  Liste d'organisations non gouvernementales en ...      True\n",
       " 6185  Thomas Drake\\n\\nThomas Andrews Drake, né le en...      True\n",
       " 6186  Alcoolisme en Russie\\n\\nL'alcoolisme en Russie...      True\n",
       " 6187  Ghetto de Kletsk\\n\\nLe Ghetto de Kletsk (en hé...      True\n",
       " 6188  Syndrome du défilé thoracobrachial\\n\\nLe syndr...      True\n",
       " 6189  Constitution tunisienne de 2014\\n\\nLa Constitu...      True\n",
       " 6190  Ultimatum de l'Union soviétique à la Lituanie\\...      True\n",
       " 6191  Énergie aux Pays-Bas\\n\\nLe secteur de l'énergi...      True\n",
       " 6192  Interstate 95 dans le Maryland\\n\\nL'Interstate...      True\n",
       " 6193  Construction agricole\\n\\nUne construction agri...      True\n",
       " 6194  Grand Prix automobile de Bahreïn 2014\\n\\nLe Gr...      True\n",
       " 6195  Royal Rumble (2014)\\n\\nL’édition 2014 du \"\" es...      True\n",
       " 6196  Le Banquet (Xénophon)\\n\\nLe Banquet (en grec a...      True\n",
       " 6197  Heidegger et la métaphysique\\n\\nOn chercherait...      True\n",
       " 6198  Bilan énergétique (statistique)\\n\\nUn bilan én...      True\n",
       " 6199  Régulation des communications électroniques\\n\\...      True\n",
       " 6200  Mémoire au Roi de Desmarets de 1709\\n\\nNicolas...      True\n",
       " 6201  Liberté du commerce et de l'industrie\\n\\nLes t...      True\n",
       " 6202  Histoire de Palerme\\n\\nL'histoire de Palerme d...      True\n",
       " 6203  Personnages de League of Legends\\n\\nLa liste d...      True\n",
       " 6204  Michel Ordener (1787-1862)\\n\\nMichel, comte Or...      True\n",
       " \n",
       " [6205 rows x 2 columns])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train, df_valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_regroup = pd.concat([df_train, df_valid])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_regroup.to_csv(PATH/'fulltrain.csv', header=None, index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LM Training\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We train two LM: one with a 60k vocabulary and one with a 30k vocabulary. The two models have different performance and computation needs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('data/lm/')\n",
    "tokenizer = Tokenizer(lang='fr', n_cpus=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm_full = (TextList.from_csv(PATH, csv_name='fulltrain.csv', cols=0, processor=[TokenizeProcessor(tokenizer=tokenizer), NumericalizeProcessor(max_vocab=60000)])\n",
    "           #Inputs: all the text files in path\n",
    "            .split_from_df(col=1)\n",
    "           #We may have other temp folders that contain text files so we only keep what's in train and test\n",
    "            .label_for_lm()           \n",
    "           #We want to do a language model so we label accordingly\n",
    "            .databunch(bs=32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm_full.save('full_lm_60k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm_full = TextLMDataBunch.load(PATH, 'full_lm_60k', bs=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60002"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_lm_full.train_ds.vocab.itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>  <col width='5%'>  <col width='95%'>  <tr>\n",
       "    <th>idx</th>\n",
       "    <th>text</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>xxbos xxmaj entrée des états-unis dans la xxmaj première xxmaj guerre mondiale \\n\\n xxup l' entrée dans la xxmaj première xxmaj guerre mondiale des états-unis se produisit en avril 1917 , après deux ans et demi d' efforts déployés par le président xxmaj woodrow xxmaj wilson pour garder les états-unis neutres dans cette guerre . xxmaj exceptée une partie xxunk soutenant les xxmaj britanniques , l' opinion publique américaine était</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>il met alors l' accent sur le caractère forcément provisoire et éphémère de la première . \\n « xxmaj si la religion est cet ensemble de vérités soumises à l’ intelligence humaine , la tentative de cette intelligence de les élucider , de les comprendre et surtout de les harmoniser intellectuellement de façon xxunk , se nomme théologie . xxmaj mais comprendre et harmoniser sont des opérations issues d’ une</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>mais xxmaj eastwood décide de réaliser le film en plus d' y interpréter le rôle du vieil entraîneur . xxmaj hilary xxmaj xxunk est embauchée pour jouer le rôle de la jeune xxunk et xxmaj morgan xxmaj freeman pour interpréter le gardien du gymnase , vieil ami de l' entraîneur . xxmaj le tournage a lieu à xxmaj los xxmaj angeles au début de l' année 2004 . xxmaj eastwood</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>abord celui qu' offrent les femmes de son foyer , son épouse et sa fille . xxmaj le statut d' infériorité qu' il leur octroie d' évidence implique que , la société accordant sa priorité aux mâles , l' aspect femelle de la nature se trouve moralement xxunk . xxmaj car les femmes ne représentent pas seulement la maternité , mais hors de cet attribut essentiel , xxmaj dombey se</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>si le grand-oncle de xxmaj sarah , xxmaj james xxmaj temple , a été reconnu coupable de régicide ) et leur loyauté à la couronne , comme celle des xxmaj churchill , se voit récompensée par un emploi à la xxmaj cour : en 1673 , xxmaj sarah a suivi sa sœur xxmaj frances à la résidence de la duchesse d' xxmaj york , xxmaj marie de xxmaj modène ,</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_lm_full.show_batch()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8XHW9//HXZyZbkybplm5J95WWpUsoSy/QgiBy2VoE4coVRQUE5ao/rz/9ea/rVXG7Ki4gICCioIAgm5SKsgjUNoUuKVBa6Jbua9Jmn8z398eclhAmnTSdmXNm5v18POYxM2fOmfPOtJlPvud7zvdrzjlEREQOJ+R3ABERCT4VCxERSUjFQkREElKxEBGRhFQsREQkIRULERFJSMVCREQSUrEQEZGEVCxERCShPL8DJMugQYPc6NGj/Y4hIpJRli5duss5V5FovawpFqNHj6ampsbvGCIiGcXMNvRkPR2GEhGRhFQsREQkIRULERFJSMVCREQSUrEQEZGEUlYszOxOM9thZrWdll1qZqvMLGpm1YfZ9lwzW21ma83sS6nKKCIiPZPKlsXdwLldltUC84Hnu9vIzMLAL4APAFOAK8xsSooyiohID6TsOgvn3PNmNrrLstcBzOxwm84C1jrn3vbWvR+4CHgtFTmb2iLc8cI6nIOoc3SeZLZrykOvdZ2K1uw96/ZE14+hd++S+L2PNluCf6+423b9WWLL4mWzQ8/N+xwPPT+4zN79Wsh7HPJeC5kRCh18boQOLvMeh0NGKGSEzQiH3n3LCxl5oRD5YSMvHLsvCIfID4coyIvd8kJ2xJ+BSLYJ4kV5lcCmTs/rgJNStbOmtg7+d+GbR7zdwe8OTWGe/cygT36YPvlhivLDlBSG6VuYR9+ifMqK8uhfXED/kgIGlhQwrLyIyv59qOpXTHlxvt/RJQcs3bCHqIMTRw9I6X6CWCzi/QkX9yvZzK4BrgEYOXJkr3Y2sKSAtd/+wKG/Urv+BemcO6q/Krvb3nWpMsksOp3fqut+jnz7nm7jul3fufivOy+fO7i8y3pdX+/8OOpcrCXoYs87nMM5R9Rbr8M5otHYeh3Rg88dkWjsvsPFHnd0OCLRKO2d7ts7orRHorR1RGmLRGmNRGlp76CprYPmttj9gdYI9c3tbNrTxN6mNuqb29/zsw8vL+K4qnKOr+rHKeMGMn1EP7VQJOl+8tc1HGiN8PD1s1O6nyAWizpgRKfnVcCWeCs6524DbgOorq7u1detmZEX7v4X+Gh/ubvbvuvy1H2H6MspHSIdUfY0tbF1Xwub9zWzaU8Tq7Y0sHJzPQtWbQdixePcY4cxf0Ylx1aW+5xYskVTWwclBan/Kg9isVgCTDCzMcBm4HLg3/yNJHJ4eeEQg0uLGFxaxAkj+r3rtX1NbfztjR08uXIb9/5zA3e+uI750yv54rmTGVpe5FNiyRaNrREGlBSkfD8pKxZmdh8wBxhkZnXA14A9wM+ACuAJM1vmnHu/mQ0H7nDOneeci5jZp4EFQBi40zm3KlU5RVKtX3EB82dUMX9GFQ0t7dz67Fvc8Y91/KV2G58+czyfOmMcoZBagNI7sZZFOOX7SeXZUFd089LDcdbdApzX6fmTwJMpiibim7KifL547mSumDWSbz/xOj9YsJpdB1r56vlT1J8hvdLUFqG4MPUHiXQFt4gPRgwo5pYrZ3D17DHc9eJ6fvLXNX5HkgzV2JrhLQsROTwz47/+9Rj2t7Tz02fWUFqUxydOG+t3LMkgHVFHc3sHxTnawS2SM0Ih47vzj+NAa4T/eeJ1xlX0Ze7kwX7HkgzR3N4BQElh6lsWOgwl4rO8cIifXD6NcRUlfOOxVbRFon5HkgzR1BoBSEvLQsVCJAAK88L89/lTWL+7ibtfWud3HMkQjW1qWYjknDmTBnPm5MHc/Mxadu5v9TuOZIBGtSxEctN//esxtLR38MMFq/2OIhmg6WDLQsVCJLeMrejLx2aP5o9LN7Gyrt7vOBJwTW1ey0KHoURyz2fOmkD/4gJ+tFCtCzk8tSxEclhZUT5Xzx7Ns6t38tqWBr/jSIC902ehloVITvr3k0dTUhDmV8+/5XcUCbCDLQsVC5EcVV6cz4dPHsVjy7ewcXeT33EkoBq9PosSjQ0lkruunj2GcMi4/YW3/Y4iAdXU2kHIoDAv9V/lKhYiATW0vIj506v4Y80mXXchcTW2RSgpyEvLiMUqFiIBds0ZY2nriHLXi7qqW96rqbUjLafNgoqFSKCNq+jL2ccM4b7FG2mNdPgdRwLmYMsiHVQsRALuipNGsrepnb++tsPvKBIwTW1qWYiI5/QJFQwrL+IPNZv8jiIB09QWScu4UKBiIRJ44ZBx6cwqXlizk837mv2OIwGSrvm3QcVCJCNcWj0C5+DBmjq/o0iANLamZ/5tULEQyQgjBhQze/xAHli6iWjU+R1HAkItCxF5j8uqR1C3t5mX3trtdxQJiMZW9VmISBfvnzqU8j756ugWAJxzsbOh1LIQkc6K8sPMm17Jgtpt1De1+x1HfNbWESUSdWkZFwpULEQyyvwZlbR1RHlq1Va/o4jPmlrTN+IsqFiIZJTjKssZM6iER17d4ncU8dmhEWfVZyEiXZkZF54wnEXrdrOtvsXvOOKjQ3NZ6ApuEYnnomnDcQ4eX6HWRS47OEueWhYiEtfYir4cX1XOI8s2+x1FfNScxlnyQMVCJCNdNK2S2s0NrN1xwO8o4pNGr1jobCgR6dYFxw8jZPDoch2KylVNXge3WhYi0q3BZUWcOm4Qf162Gec0/EcuamxVy0JEeuDCacPZsLuJZZv2+R1FfKCWhYj0yLnHDqUgHOLxFbpALxc1HrooTy0LETmMsqJ85kyq4PEVWzQSbQ5qaotQmBciHLK07E/FQiSDXXDCcLY3tLJk/R6/o0iaNbZF0tZfASksFmZ2p5ntMLPaTssGmNlCM1vj3ffvZtsOM1vm3R5NVUaRTHfWMYPpkx/mMV2gl3OaWtM34iyktmVxN3Bul2VfAp5xzk0AnvGex9PsnJvm3S5MYUaRjFZckMf7pgzhyZXbiHRE/Y4jadTYFknb1duQwmLhnHse6No2vgj4jff4N8DFqdq/SK44//hh7Gls06RIOaaprSNt40JB+vsshjjntgJ494O7Wa/IzGrMbJGZqaCIHMYZEysoLczTBXo5Jjalaha0LI7SSOdcNfBvwE/MbFy8lczsGq+o1OzcuTO9CUUCoig/zDlTh7KgdhutkQ6/40iaxKZUzd6WxXYzGwbg3e+It5Jzbot3/zbwLDC9m/Vuc85VO+eqKyoqUpNYJANccMIw9rdGeHa1/mjKFU1tHdlxNlQ3HgWu8h5fBfy56wpm1t/MCr3Hg4DZwGtpSyiSgWaPH8SAkgIeXaZDUbmiqS1LWhZmdh/wMjDJzOrM7OPATcDZZrYGONt7jplVm9kd3qbHADVmthz4O3CTc07FQuQw8sMhLjxhOAtf2675uXNEY2t6WxYp25Nz7opuXjorzro1wCe8xy8Bx6Uql0i2umRGFXe/tJ7HV27hwyeN8juOpFBH1NHcnj3XWYhIGh1bWcbEIX350yuaFCnbNbd7I87qbCgROVJmxvwZVSzdsJd1uxr9jiMp1ORNqdpHLQsR6Y2Lp1USMnj4lTq/o0gKvTNLnoqFiPTC0PIiZo8fxJ9e3ayRaLNYY+vBuSx0GEpEeumSGVXU7W3WSLRZTH0WInLU3j91KCUFYR7SoaisdahlocNQItJbfQrC/Ovxw3hixdZDXyqSXZra1LIQkSS4rHoEjW0dPLlSU65mo3f6LNSyEJGjMHNUf8YOKuGBGh2KykaHWhZZPDaUiKSBmXFp9QgWr9/D2zsP+B1HkqyxTS0LEUmSS2ZUEg4ZDy5V6yLbNLV2EA4ZhXnp+wpXsRDJUoPLijhjYgUPvVJHh665yCqN3oizZpa2fapYiGSxy6qr2N7QyvNrNM9FNmlqTe8seaBiIZLVzpw8hAElBTxQs8nvKJJEjWmeywJULESyWkFeiHnTK1n42nZ2H2j1O44kSXNbR1ovyAMVC5Gsd/mJI2jvcBq6PIvEWhY6DCUiSTRhSCkzR/XnviUbcU4d3dmgqa2DEh2GEpFk+9CJI3h7ZyM1G/b6HUWSoLE1QnEaL8gDFQuRnHD+8cMoLczjvsUb/Y4iSaCWhYikRHFBHhdOG86TK7dS39zudxw5So2t6rMQkRS5YtZIWtqj/HmZOrozWTTqONAaoa8OQ4lIKhxbWc7U4WXct3iTOroz2IG2CFEH5X3y07pfFQuRHHL5rJG8vrWB5XX1fkeRXmrwDiOqWIhIylw8bTglBWHueWm931Gklw72OZWpWIhIqpQW5fPBmVU8vmIru3RFd0aqV8tCRNLh308ZTVtHlPt1Gm1GajjUslAHt4ik0PjBfTltwiDuXbSR9o6o33HkCDU0xyY+UstCRFLuqlNGs62hhadXbfc7ihwhHYYSkbSZO3kwVf378JuX1/sdRY5QfXM7IUPXWYhI6oVDxkdOGcXidXuo3azTaDNJfXM7ZX3y0zpLHqhYiOSsy6pHUFaUx9V3L2GlrrvIGA0t7Wk/BAUqFiI5q19xAX+87hTywyEu+9XLPL1qm9+RpAfqm1UsRCTNJg8t4+EbTmXikL5ce+9S/rhE068GnYqFiPhicGkR919zCrNGD+A7f3mdpraI35HkMOqb2ykrUrEQER/0KQjzxXMnsa+pXa2LgGtojqR9qA9QsRARz8xRA6ge1Z/bX1hHRBfrBZJzjoZsOwxlZnea2Q4zq+20bICZLTSzNd59/262vcpbZ42ZXZWqjCLybteeMY7N+5p5YuVWv6NIHC3tUdo6otlVLIC7gXO7LPsS8IxzbgLwjPf8XcxsAPA14CRgFvC17oqKiCTXWZMHM66ihNuef1tzXgRQvU/jQkEKi4Vz7nlgT5fFFwG/8R7/Brg4zqbvBxY65/Y45/YCC3lv0RGRFAiFjGtOH8uqLQ28uHa333GkC7+G+oAeFgszG2dmhd7jOWZ2o5n168X+hjjntgJ494PjrFMJdO5hq/OWxct1jZnVmFnNzp07exFHRLq6eHolFaWF/Or5t/yOIl00tAS8WAAPAR1mNh74NTAG+H2KMsW7hj1ue9g5d5tzrto5V11RUZGiOCK5pTAvzNWzx/DCml2sqNvndxzppL4p+MUi6pyLAPOAnzjnPgcM68X+tpvZMADvfkecdeqAEZ2eVwFberEvEemlK08eSVlRHr/4+1q/o0gnh/osAnydRbuZXQFcBTzuLetN2ke998C7/3OcdRYA55hZf69j+xxvmYikSWlRPh89dTQLVm3nze37/Y4jnsD3WQAfA04Bvu2cW2dmY4B7D7eBmd0HvAxMMrM6M/s4cBNwtpmtAc72nmNm1WZ2B4Bzbg/wLWCJd/umt0xE0uhjs8dQXBDml2pdBMbBPgs/Lsrr0flXzrnXgBsBvL/2S51zNyXY5opuXjorzro1wCc6Pb8TuLMn2UQkNfqXFPDhk0by63+s4/NnT2LkwGK/I+W8+uZ2SgvzCIfSOzw59PxsqGfNrMy7BmI5cJeZ/W9qo4mI3z5x2ljyQiFueU5nRgXBwbks/NDTw1DlzrkGYD5wl3NuJvC+1MUSkSAYUlbEpdVVPLS0jq31zX7HyXkNGVAs8ryzly7jnQ5uEckB150xjqhz3PKsWhd+a2iOUO7D1dvQ82LxTWJnJL3lnFtiZmOBNamLJSJBMWJAMZdWj+D+xZvYsk+tCz/5NZcF9LBYOOcecM4d75z7lPf8befcJamNJiJBccPccTjUuvCbX3NZQM87uKvM7GFvFNntZvaQmVWlOpyIBENV/1jr4g9L1LrwU+BbFsBdxC6oG05snKbHvGUikiNumDseh+OXz+q6Cz+0RaI0t3cEvlhUOOfucs5FvNvdgAZjEskhlf36cJnXutis1kXaHRpEsDjYxWKXmV1pZmHvdiWg8YtFcswNc8cDcItaF2nn57hQ0PNicTWx02a3AVuBDxIbAkREcsjwfn24tHoEf1xSx7b6Fr/j5BQ/x4WCnp8NtdE5d6FzrsI5N9g5dzGxC/REJMd8yrvu4lZd1Z1WDc3+jQsFRzdT3ueTlkJEMsaIAcXMn1HJfYs3smO/Whfp8k7LItgX5cWT/pGsRCQQrp8znvaOKLc//7bfUXJGJrcsNJu7SI4aPaiEi6dVcu+ijew+0Op3nJwQ6D4LM9tvZg1xbvuJXXMhIjnq+rnjaYl0cPsL6/yOkhMaWiIU5YcozAv7sv/DFgvnXKlzrizOrdQ558+BMxEJhPGD+3LB8cO55+X1al2kQX2Tf0N9wNEdhhKRHHfjWeNpbu/gthfUd5Fqfg71ASoWInIUxg8u5aIThnPPSxvYpdZFSqlYiEhGu/GsCbRGOrhNZ0allIqFiGS0sRV9uXhaJfe8vJ6d+9W6SJWGFv9myQMVCxFJgs+cNYH2DsevdFV3yqhlISIZb4x33cVvF21gR4Ou6k62jqhjf0tELQsRyXw3njWeSNRxi1oXSbe/xd8L8kDFQkSSZNTAEj44o4rf/XMjW+s130UyNTRHACgr8u/yNhULEUmaT585Huccv/y7WhfJtLepDYB+xQW+ZVCxEJGkGTGgmMuqR3D/ko2aTS+Jtnv9QEPKCn3LoGIhIkl1w9zxGMbP/6bZ9JLlYLEYWlbkWwYVCxFJquH9+nDFrBE8ULOJDbsb/Y6TFbbWtxAOGQP7qmUhIlnkhrnjyQ+H+NHTb/odJStsa2hhSGkh4ZB/0wipWIhI0g0uK+Lj/zKGR5dvoXZzvd9xMt72hhaGlPt3CApULEQkRa45Yyz9i/P53lNv+B0l422rb/G1vwJULEQkRcqK8rlh7nheWLOLF9fu8jtORttW38JQtSxEJFtdefIohpcX8b2n3sA5zcTcG/tb2mls61DLQkSyV1F+mM+fM4kVdfU8sXKr33Ey0qHTZtWyEJFsNm96JZOHlvL9p1bTGunwO07G2Vp/8II8FQsRyWLhkPHl845h454m7l200e84GWebVyyG5WLLwsz+w8xqzWyVmX02zutzzKzezJZ5t6/6kVNEkuOMiRWcNmEQNz+zhvqmdr/jZJR3hvrIsWJhZscCnwRmAScA55vZhDirvuCcm+bdvpnWkCKSdF/+wDE0tLTzi2c1DMiR2NbQQr/ifIryw77m8KNlcQywyDnX5JyLAM8B83zIISJpNGV4GZfMqOLuF9ezaU+T33EyRhCusQB/ikUtcLqZDTSzYuA8YESc9U4xs+Vm9hczmxrvjczsGjOrMbOanTt3pjKziCTB/zlnIqEQ/PDp1X5HyRjbGvy/xgJ8KBbOudeB7wELgaeA5UCky2qvAKOccycAPwMe6ea9bnPOVTvnqisqKlKYWkSSYVh5H66ePYY/L9MwID21rb41Z1sWOOd+7Zyb4Zw7HdgDrOnyeoNz7oD3+Ekg38wG+RBVRJLs2jPG0a84n+8vUOsikbZIlF0HWnOzZQFgZoO9+5HAfOC+Lq8PNTPzHs8ilnN3unOKSPKV98nn03PH8/ybOzUMSAI79vs/j8VBfl1n8ZCZvQY8BtzgnNtrZteZ2XXe6x8Eas1sOXAzcLnTWAEiWePgMCA3/eUNolH9anfn0GmzAWhZ+DL7t3PutDjLbu30+OfAz9MaSkTS5uAwIF94YDlP1m7l/OOH+x0pkLbVtwK53bIQkRw3b3olk4aU8oMFq2mLRP2OE0hb62PzmPt99TaoWIiIT2LDgExmw+4m7nl5vd9xAml7QwuFeSHK++T7HUXFQkT8M2fSYE6fWMHNz6xhb2Ob33ECZ1tD7Ewo73wfX6lYiIivvnLeMRxojfDTZ9YkXjnHbKtvDkR/BahYiIjPJg0t5fJZI7l30Qbe2nnA7ziBEpSrt0HFQkQC4HPvm0hRfpjvPvm631ECwznH9oZgXL0NKhYiEgAVpYVcP3ccf319B8+9qXHeAPY2tdMWiaplISLS2cf/ZQxjBpXw9UdXaUY93jltVi0LEZFOCvPCfO2CKazb1cgdL6zzO47vgnT1NqhYiEiAzJk0mHOnDuVnf1vD5n3Nfsfx1cG5t9WyEBGJ478vmALAtx57zeck/lqz/QDFBWHfp1M9SMVCRAKlsl8fPnPmBJ5atS2nO7trN9czdXgZ4ZD/F+SBioWIBNAnTot1dn/j0VU5OW5UR9SxaksDU4eX+x3lEBULEQmcwrwwX71gCm/vauSuF3Ovs3vdrgM0t3dwbKWKhYjIYc2dNJj3HTOYm59Zc+jMoFxRu7kBgONULEREEvvv86fQHnU5d2X3ys31FOaFGFdR4neUQ1QsRCSwRg0s4drTx/LIsi0sXrfH7zhpU7u5nmOGlZEXDs5XdHCSiIjEcf2c8VT268NX/1xLpCP7O7ujUcdrWxoCdQgKVCxEJOD6FIT57/On8Ma2/dz90nq/46Tchj1N7G+NcGxlmd9R3kXFQkQC7/1ThzB3UgU/Xvgm2+qzu7O7dnM9QKBOmwUVCxHJAGbG1y+cSnvU8T9PZPeV3bVb6ikIh5g4pNTvKO+iYiEiGWHUwBJumDOex1ds5R9rdvkdJ2VqN9czaWgpBXnB+noOVhoRkcO49oyxjB5YzH89spLmtuwbxtw5R+3mhsD1V4CKhYhkkKL8MN+Zdxzrdzfx47++6XecpKvb20x9c3ugrtw+SMVCRDLKqeMHccWskdzxwtu8unGv33GS6mDn9rEB69wGFQsRyUBfPm8yQ8qK+OKDK7JqVr3aLfXkhYxJQ4PVuQ0qFiKSgcqK8vnOvONYs+MAv/jbWr/jJE3N+r1MGlpKUX7Y7yjvoWIhIhlp7uTBzJ9eyS+efYtlm/b5Heeo1Te1U7NhL3MmVfgdJS4VCxHJWF+7YCpDSgv57P2v0tga8TvOUXluzU46oo4zJw/xO0pcKhYikrHKi/P53w9NY8OeJr6Z4dOw/v2NHQwoKWDaiH5+R4lLxUJEMtrJYwfyqTPG8YeaTTxVu9XvOL3SEXX8ffUO5kyqCMw0ql2pWIhIxvvs+yZyfFU5X/rTSrbWN/sd54i9unEv+5raOXPyYL+jdEvFQkQyXkFeiJ98aBrtkSifuveVjDud9pk3dpAXMk6bEMzObVCxEJEsMbaiLz+89ASWbdqXcf0Xf3t9ByeOHkB5n3y/o3RLxUJEssYHjhvGtWeM5Xf/3MgDNZv8jtMjm/Y0sXr7fs46JriHoEDFQkSyzH+eM4lTxw3kK4/UsrKu3u84Cf199Q6AQPdXgE/Fwsz+w8xqzWyVmX02zutmZjeb2VozW2FmM/zIKSKZJy8c4mdXTGdQSQHX/raGXQda/Y50WM+8voMxg0oYW9HX7yiHlfZiYWbHAp8EZgEnAOeb2YQuq30AmODdrgFuSWtIEcloA/sW8qt/r2Z3YxvX/+4V2gM6d/fexjZeemsXZwW8VQH+tCyOARY555qccxHgOWBel3UuAu5xMYuAfmY2LN1BRSRzHVdVzvcuOZ7F6/bwrceD2eH92IottHc45s2o9DtKQn4Ui1rgdDMbaGbFwHnAiC7rVAKde6fqvGXvYmbXmFmNmdXs3LkzZYFFJDNdPL2ST542hnte3sD9izf6Hec9Hlpax+ShpYGbbzuetBcL59zrwPeAhcBTwHKg66Au8S5hdHHe6zbnXLVzrrqiIrjnJ4uIf/7vuZM5bcIgvvJILU/VbvM7ziFrd+xneV09H5xZ5XeUHvGlg9s592vn3Azn3OnAHmBNl1XqeHdrowrYkq58IpI98sIhbr1yJsdVlnPjfa/ywppgHIV4cOlmwiHjomnBPwQF/p0NNdi7HwnMB+7rssqjwEe8s6JOBuqdc5k56IuI+K6kMI+7P3YiYytKuOaepSzdsMfXPB1Rx8Ov1jFnYgUVpYW+Zukpv66zeMjMXgMeA25wzu01s+vM7Drv9SeBt4G1wO3A9T7lFJEs0a+4gN9+/CSGlhfx0TuXsOjt3b5leXHtLrY3tHJJhhyCAv8OQ53mnJvinDvBOfeMt+xW59yt3mPnnLvBOTfOOXecc67Gj5wikl0qSgv5/SdPYkh5ER+5c7FvfRgPvVJHeZ/8wF+13Zmu4BaRnDKsvA8PXHsKU4eXcf3vlnJfms+Sqm9qZ8GqbVxwwjAK84I3fWp3VCxEJOf0Lyngd584idMnVvDlP63kR0+vxrn3nHCZEj/+65u0RaJcefKotOwvWVQsRCQnFRfkcftHqvlQ9Qh+9re1fO4Py1I+tPkb2xr47aINfPikUUweWpbSfSVbnt8BRET8kh8OcdMlxzFyYDE/WLCaLftauO0jM+lXXJD0fTnn+NqfV1FalMf/OWdi0t8/1dSyEJGcZmbcMHc8P718Gss27WP+LS+xaU9T0vfz+Iqt/HPdHr5wzqSUFKNUU7EQEQEumlbJvZ84id0H2pj3yxdZUbcvae/d1BbhO0++ztThZVwxa2TS3jedVCxERDyzxgzgoU+dSlF+mA/9ahELVh39qbXOOf7fn1aytb6Fb1w4lXAo3mhGwadiISLSyfjBfXn4+tlMHNKXa3+7lG89/hptkd4Pcf7jhW/yyLItfOGciVSPHpDEpOmlYiEi0kVFaSF/vO4UPnrqaH79j3V88NaXWL+r8Yjf54GaTdz8t7VcVl3FDXPHpyBp+li6zi1OterqaldTowu9RSS5nqrdxhcfXE5DS4TJQ0s5eexAZo8fxNxJFeSF4/+97ZzjiZVb+ez9yzh57EDu+tiJ5Hezrt/MbKlzrjrRejp1VkTkMM49dijHV5Xz8KubWfT2bu5fspG7X1pPZb8+XP0vY7j8xBGUFMa+Sp1zvLBmFz9a+CbLN+1j6vAyfnnljMAWiiOhloWIyBFoi0R57s2d3P782yxev4fSwjwqygqJRh2tkShb61uo7NeHG88az/wZVYEvFGpZiIikQEFeiLOnDOHsKUN4deNe7l+8iQNtEcJmhEPGzFH9ubS6KqPGfeoJFQsRkV6aPrI/00f29ztGWgS7fSQiIoGgYiEiIgmpWIiISEIqFiIikpCKhYiIJKRiISIiCalYiIhIQioWIiKSUNYM92FmO4EN3tNyoL7Tywefd17eddkgYNcR7LIH1gd9AAAH+0lEQVTrPhK91l2m7h6nO9/hMsXLFW9Zrn+Gh8sXL1e8ZfoM9RmmO98o4Grn3GOHTeGcy7obcFu8552Xd10G1BzNPhK91l2mHuRKS77DZdJnePT59BnqMwxqvkT7OHjL1sNQXSvkY3GWx1t2NPtI9Fp3mbp7nO58h8vUXR59hodfps9Qn2G8+yOV6nyJ9gFk0WGoo2VmNa4HIy/6Jej5IPgZg54Pgp8x6Pkg+BmDnq872dqy6I3b/A6QQNDzQfAzBj0fBD9j0PNB8DMGPV9calmIiEhCalmIiEhCWVcszOxOM9thZrW92Hamma00s7VmdrOZmbf8D2a2zLutN7NlQcvovfYZM1ttZqvM7PtBymdmXzezzZ0+x/N6my9VGTu9/gUzc2Y2KEj5zOxbZrbC+/yeNrPhvc2Xwow/MLM3vJwPm1m/gOW71Pv9iJpZr/oNjiZXN+93lZmt8W5XJfoZfHMkp3Blwg04HZgB1PZi28XAKYABfwE+EGedHwFfDVpGYC7wV6DQez44YPm+Dnwh6P/OwAhgAbFrdgYFKR9Q1mmdG4Fbg/YZAucAed7j7wHfC1i+Y4BJwLNAdTpzefsc3WXZAOBt776/97h/ov+nftyyrmXhnHse2NN5mZmNM7OnzGypmb1gZpO7bmdmw4j9Mr7sYv9S9wAXd1nHgMuA+wKY8VPATc65Vm8fOwKWL6lSmPHHwBeBo+rMS0U+51xDp1VLAprxaedcxFt1EVAVsHyvO+dW9zbT0eTqxvuBhc65Pc65vcBC4Nx0/i71VNYVi27cBnzGOTcT+ALwyzjrVAJ1nZ7Xecs6Ow3Y7pxbE8CME4HTzOyfZvacmZ0YsHwAn/YOT9xpZqmYi/KoMprZhcBm59zyFGQ76nxexm+b2Sbgw8BXg5ixk6uJ/UUc1HzpzhVPJbCp0/ODWf34GQ4r6+fgNrO+wKnAA50O+RXGWzXOsq5/uV3BUbYq4klSxjxizdiTgROBP5rZWO+vkiDkuwX4lvf8W8QO5119tNmSldHMioGvEDuMknTJ+n/onPsK8BUz+zLwaeBrQcvovddXgAjwuyDmS6bD5TKzjwH/4S0bDzxpZm3AOufcvMNkTevP0BNZXyyItZ72OeemdV5oZmFgqff0UWJfZp2bzFXAlk7r5wHzgZkBzVgH/MkrDovNLEpsDJqdQcjnnNveabvbgceTkCuZGccBY4Dl3i98FfCKmc1yzm0LQL6ufg88QRKLRbIyep205wNnJeOPlWTnS4G4uQCcc3cBd3k5nwU+6pxb32mVOmBOp+dVxPo26kjvz5CYnx0mqboBo+nU+QS8BFzqPTbghG62W0LsL/ODHUrndXrtXOC5oGYErgO+6T2eSKxpawHKN6zTOp8D7g/aZ9hlnfUcRQd3ij7DCZ3W+QzwYNA+Q+/35DWg4mizpfLfmKPo4O5tLrrv4F5H7KhAf+/xgJ7+P03nzbcdp+wHih0m2gq0E6vOHyf2F+NTwHLvP3Lcs5mAaqAWeAv4OZ2+bIG7geuCmhEoAO71XnsFODNg+X4LrARWEPvrb1hv86Xy37nTOus5urOhUvEZPuQtX0FsLJ/KoH2GwFpif6gs8269PmMrRfnmee/VCmwHFqQrF3GKhbf8au9zWwt87Ej+n6bzpiu4RUQkoVw5G0pERI6CioWIiCSkYiEiIgmpWIiISEIqFiIikpCKhWQ1MzuQ5v3dYWZTkvReHRYbYbbWzB6zBCO4mlk/M7s+GfsW6UqnzkpWM7MDzrm+SXy/PPfOQHkp1Tm7mf0GeNM59+3DrD8aeNw5d2w68kluUctCco6ZVZjZQ2a2xLvN9pbPMrOXzOxV736St/yjZvaAmT0GPG1mc8zsWTN70GJzN/zOG5EYb3m19/iAN/DfcjNbZGZDvOXjvOdLzOybPWz9vMw7Ax72NbNnzOwVi813cJG3zk3AOK818gNv3f/09rPCzL6RxI9RcoyKheSinwI/ds6dCFwC3OEtfwM43Tk3ndiIrt/ptM0pwFXOuTO959OBzwJTgLHA7Dj7KQEWOedOAJ4HPtlp/z/19p9wvB9v7KOziF35DtACzHPOzSA2j8mPvGL1JeAt59w059x/mtk5wARgFjANmGlmpyfan0g8uTCQoEhX7wOmdBohtMzMSoFy4DdmNoHYCJ/5nbZZ6JzrPIfBYudcHYDFZk4cDfyjy37aeGfAxKXA2d7jU3hnboLfAz/sJmefTu+9lNhcBxAbK+g73hd/lFiLY0ic7c/xbq96z/sSKx7Pd7M/kW6pWEguCgGnOOeaOy80s58Bf3fOzfOO/z/b6eXGLu/R2ulxB/F/l9rdO52C3a1zOM3OuWlmVk6s6NwA3ExsLosKYKZzrt3M1gNFcbY34LvOuV8d4X5F3kOHoSQXPU1sLggAzOzg0NLlwGbv8UdTuP9FxA5/AVyeaGXnXD2xaVS/YGb5xHLu8ArFXGCUt+p+oLTTpguAq735FjCzSjMbnKSfQXKMioVku2Izq+t0+zyxL95qr9P3NWLDuwN8H/iumb0IhFOY6bPA581sMTAMqE+0gXPuVWIjml5ObEKhajOrIdbKeMNbZzfwoneq7Q+cc08TO8z1spmtBB7k3cVEpMd06qxImnmz8jU755yZXQ5c4Zy7KNF2In5Sn4VI+s0Efu6dwbSPJE4vK5IqalmIiEhC6rMQEZGEVCxERCQhFQsREUlIxUJERBJSsRARkYRULEREJKH/D2bxU6LMOkiSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = language_model_learner(data_lm_full, drop_mult=0, qrnn=True, callback_fns=ShowGraph)\n",
    "learn.lr_find()\n",
    "learn.recorder.plot(skip_start=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 14:32:14\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      3.874567    3.998646    0.304150  (1:27:11)\n",
      "2      3.636466    3.846859    0.310775  (1:27:01)\n",
      "3      3.593106    3.822112    0.312760  (1:27:02)\n",
      "4      3.584104    3.792033    0.314823  (1:27:45)\n",
      "5      3.611241    3.744764    0.319751  (1:27:08)\n",
      "6      3.562205    3.699586    0.325680  (1:27:09)\n",
      "7      3.522145    3.659924    0.331840  (1:27:06)\n",
      "8      3.563134    3.609655    0.339367  (1:27:27)\n",
      "9      3.532401    3.576437    0.345281  (1:27:15)\n",
      "10     3.486668    3.560602    0.347943  (1:27:00)\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt8VdWd9/HP71xyAwIh3AKIiYhyEwEjQrWW1noDb628OjhqW6dTW33Gqp2nLdqxtVPb2svTTnmsttY6M/Vx7Fi0rYOKikqpVbFBELmIXAQJBBIuCQFyknNZzx9nJyQhCYGcnJNkf9+v13mdvde+rLV4HfI9e++19zHnHCIi4l+BTDdAREQyS0EgIuJzCgIREZ9TEIiI+JyCQETE5xQEIiI+pyAQEfE5BYGIiM8pCEREfC6Uzsry8gvcxDPGprNKEZFeb+XKlXudc0O7a/9pDYJBw0ZSVlaWzipFRHo9M9venftP66khPdVIRKTn0TUCERGfUxCIiPhcWq8RiIicqGg0Snl5OZFIJNNN6XY5OTmMHj2acDic1noVBCLSo5WXlzNgwACKi4sxs0w3p9s459i3bx/l5eWUlJSkte70nhrS1WIROUGRSITCwsI+HQIAZkZhYWFGjnx0jUBEery+HgKNMtVPDR8VEfE5HRGIiHSgurqaBx988IS3mzNnDtXV1d3QotRTEIiIdKC9IIjH4x1u99xzzzFo0KDualZKadSQiEgHFixYwJYtW5g6dSrhcJj+/ftTVFTE6tWrWb9+Pddccw07duwgEolw++23c/PNNwNQXFxMWVkZhw4d4vLLL+eCCy7g9ddfZ9SoUfzpT38iNzc3wz07SkEgIr3Gd/5nHet3HUzpPieOzOfbV05qd/n999/P2rVrWb16NcuWLWPu3LmsXbu2aYjno48+yuDBg6mrq+Pcc8/l2muvpbCwsMU+Nm3axBNPPMGvf/1rPvOZz/DUU09xww03pLQfXaEgEBE5ATNmzGgxzn/hwoX84Q9/AGDHjh1s2rTpmCAoKSlh6tSpAJxzzjls27Ytbe3tDAWBiPQaHX1zT5d+/fo1TS9btoylS5fyxhtvkJeXx+zZs9u8DyA7O7tpOhgMUldXl5a2dtZxLxab2aNmVmlma5uVDTazl8xsk/de0JnKnAaQikgvM2DAAGpra9tcVlNTQ0FBAXl5ebz33nu8+eabaW5danRm1NB/AJe1KlsAvOycGwe87M2LiPQ5hYWFnH/++UyePJmvfe1rLZZddtllxGIxpkyZwj333MPMmTMz1MquMeeO/y3dzIqBxc65yd78RmC2c67CzIqAZc65M4+3n+FjJ7o9W9Z3rcUi4isbNmxgwoQJmW5G2rTVXzNb6Zwr7a46T/Y+guHOuQoA731Y6pokIiLp1O03lJnZzWZWZmZlkUh9d1cnIiIn6GSDYI93SgjvvbK9FZ1zDzvnSp1zpTnNrpyLiEjPcLJB8AzwOW/6c8CfUtMcERFJt84MH30CeAM408zKzewLwP3AxWa2CbjYmxcRkV7ouDeUOeeua2fRRSlui4iIZIB+j0BEJIX69+8PwK5du5g3b16b68yePZuysrJ0NqtDegy1iEg3GDlyJIsWLcp0MzpFzxoSEenAN77xDU499VRuvfVWAO69917MjOXLl3PgwAGi0Sj33XcfV199dYvttm3bxhVXXMHatWupq6vjpptuYv369UyYMKHHPWtIQSAivcfzC2D3u6nd54iz4PL2x7vMnz+fO+64oykInnzySZYsWcKdd95Jfn4+e/fuZebMmVx11VXt/ubwQw89RF5eHmvWrGHNmjVMnz49tX3oIgWBiEgHpk2bRmVlJbt27aKqqoqCggKKioq48847Wb58OYFAgJ07d7Jnzx5GjBjR5j6WL1/OV77yFQCmTJnClClT0tmF41IQiEjv0cE39+40b948Fi1axO7du5k/fz6PP/44VVVVrFy5knA4THFxcZuPn26uvaOFnkAXi0VEjmP+/Pn87ne/Y9GiRcybN4+amhqGDRtGOBzm1VdfZfv27R1uf+GFF/L4448DsHbtWtasWZOOZneajghERI5j0qRJ1NbWMmrUKIqKirj++uu58sorKS0tZerUqYwfP77D7W+55RZuuukmpkyZwtSpU5kxY0aaWt45nXoMdaoMO22iq9yqx1CLSOfpMdQ99zHUJyWNmSMiIp2kawQiIj6nIBCRHi+dp7AzKVP9VBCISI+Wk5PDvn37+nwYOOfYt28fOTk5aa9bo4ZEpEcbPXo05eXlVFVVZbop3S4nJ4fRo0envV4FgYj0aOFwmJKSkkw3o0/TqSEREZ9LcxD07XN8IiK9kY4IRER8Tr9QJiLiczoiEBHxOQWBiIjPpTcIdG5IRKTH0RGBiIjPKQhERHxOQSAi4nMaPioi4nM6IhAR8TkFgYiIzykIRER8TkEgIuJzCgIREZ/rUhCY2Z1mts7M1prZE2aW/t9YExGRLjnpIDCzUcBXgFLn3GQgCMxPVcNERCQ9unpqKATkmlkIyAN2db1JIiKSTicdBM65ncBPgA+BCqDGOfdix9ucbG0iItJdunJqqAC4GigBRgL9zOyGNta72czKzKwsGo2efEtFRKRbdOXU0CeBD5xzVc65KPA08JHWKznnHnbOlTrnSsPhcBeqExGR7tCVIPgQmGlmeWZmwEXAhtQ0S0RE0qUr1whWAIuAt4F3vX09nKJ2iYhImoS6srFz7tvAt1PUFhERyQDdWSwi4nNpDgKNHxUR6Wl0RCAi4nP6hTIREZ/TEYGIiM8pCEREfE5BICLicwoCERGfUxCIiPhcekcNadiQiEiPoyMCERGfUxCIiPicgkBExOcUBCIiPqcgEBHxOQWBiIjPKQhERHxOQSAi4nN6DLWIiM/piEBExOcUBCIiPqcgEBHxufQGgS4SiIj0OGkNgkgsns7qRESkE3RqSETE59IaBOGgckdEpKdJ61/maDyRzupERKQT0v4VXWEgItKzpD0IKmvr012liIh0IO1BUFFdl+4qRUSkA2kPguXvV6W7ShER6UCXgsDMBpnZIjN7z8w2mNms422je8pERHqWUBe3/zmwxDk3z8yygLzjbXDWqIFdrFJERFLppIPAzPKBC4HPAzjnGoCGTmx3slWKiEg36MqpodOAKuDfzWyVmT1iZv1ar2RmN5tZmZmVAfxx9c4uVCkiIqnWlSAIAdOBh5xz04DDwILWKznnHnbOlTrnSgGeXVPRhSpFRCTVuhIE5UC5c26FN7+IZDCIiEgvctJB4JzbDewwszO9oouA9SlplYiIpE1XRw3dBjzujRjaCtzU9SaJiEg6dSkInHOrgdIT3S6RcAQCGj0kItITpPXO4pD3x39z1aF0VisiIh1IaxCMGJgDwOH6WDqrFRGRDqQ1CLJCyepqIwoCEZGeIq1BEI8nnzT02UffSme1IiLSgfT+VGVIP1UpItLTpPUvc2442DTtnJ5DKiLSE2TsK/reQ8d9Pp2IiKRBxoJg057aTFUtIiLNZCwI/v6RFcdfSUREup2u3oqI+Fzag2DDv17WNN0QS6S7ehERaSXtQZCbdXTk0Bn/8ny6qxcRkVZ0akhExOcyEgSLb7ugaXrl9gOZaIKIiHgyEgSTRw1smr72odeJJ3RzmYhIpmTs1NC4Yf2bprfqsdQiIhmTsSA4//QhTdMX/2y5RhCJiGRIxoLg21dObDH/3cX6uWMRkUzIWBCYGdfNGNM0/9ib21n1oS4ci4ikW0aHj37vmskt5j/14OsZaomIiH9lNAgCAWPr9+e0KFu7syZDrRER8aeM31AW8H7QvtEV//c1DkaiGWqNiIj/ZDwIALbdP7fF/JR7X8xQS0RE/KdHBAHAqnsubjFfvODZDLVERMRfekwQFPTLOqZsxdZ9+klLEZFu1mOCAOCbcya0mP+7h9/ki78to64hnqEWiYj0fT0qCL544WnHlC3dUMmEby3JQGtERPyhRwUBHHutoFHxgmd5t1xDS0VEUq3HBUFBvyy23T+XBZePP2bZlQ+8RlVtvZ5LJCKSQj0uCBrNmVzUZvm531vKGf/yPBU1dWlukYhI39TlIDCzoJmtMrPFqWhQozGFedzb6sF0zc36wSsseGoNxQueJRLVxWQRkZOViiOC24ENKdjPMT5/fgkr7r6o3eW/+9sOAMbfs4S9h+p5c+s+ItG47kwWETkBoa5sbGajgbnA94CvpqRFrQzPz2Hb/XO5b/F6Hnntg3bXK71v6TFl55UM5r5rJjNu+IBkQTwGgSCYHbOuiIhfdSkIgH8Dvg4MSEFbOvQvV0zsMAjasuKD/Vz8s+VN89s+VQ4v3gP9hkL/odBvWLP3YV75sKPzOYMg0GMvo4iIpMRJB4GZXQFUOudWmtnsDta7GbgZYMyYMe2t1imvfePjrN5RzT/916qT2v62ZQm+PenzDLGDVOz8kIGRHeRVvAOHq8C1cZ0hEEqGQ1NADG0VFs1CJK8webQhItLL2Mk+wsHMfgDcCMSAHCAfeNo5d0N725SWlrqysrKTqq+1ldsP8NL6Pfzyz1u6vK+ffeYsfviHNygtjDN3bIhLTw0QPbiHcF0VgSNVcKgKDlfCoSrc4Uos3nDsTiyQDIMOjzK88n5DIBjucrtFxB/MbKVzrrTb9p+KZ/l4RwT/2zl3RUfrpTIIGu0/3MCM7y0lluieZxKFg0Y07njiizOZNbaQ4gWLmVDgeP4L4+FQZfJo4nCVN13ZLDS8ZdEjbe84mJU84rBg8kgiEPJereYt2HZ5INhsWVvLQ8lwaj7ffJ3GbYONy8LHzgfD3vptLGt3vnG69bbhZHt0fUbkhCkIOiGRcDxZtoNPTx9NVih5Tn/j7lru+eNa3tq2P+X1taVkSD9q6qLcPWcC//v37wDw3asnccnpAxjkDhCu20vgyF7e27wFd7iSCUOycIkY8ViUEInkqalEDBIxDh2pp7YuwogBYSwRg4S3rGmd1u/etLfceWWWSLRc3mIfsbT8uxyjRVB4QRLMglAWhHK86exm79knuMwra7GsVVkwu9l+cpIBJtKD9Yog6KzuCoLO+PJjK1mybndG6m7LR8cN4S+b9jbN/+rGc7h00giqaus593stR0Atvu0CJo8a2DS/pryaXdURzho9kLJt+zlzxAA2Vx7i/LFDmLPwL1TURIBjf+fhml/8ldU7qgFYd+8l9MsKJAMhHoVENBkW8Sg799cyon+IIMn5P79XQcWBWq6dOoIwcWqO1DEwC4jHOByJkGVxwua8/cRa7ItEjHXl+zAXY+KwPG9ZLDmCq3HdeBRiEYjVQ7yhjfcIxBogXt9yWVvXdU6GBZqFROvAaB4c4baXNQZZu+tntxFiWW0HWzB8dJkGKohHQZAizjlK7nouI3WnSjBgxE/gFNiVZ4/kf97Z1e7yu+eM5/09h/ifd3ZxyuA8/vGCEhY8/W6LdW6YOYb/9+aHx2z79+eN4frzxjB34WtNZU/dMouh/XP4Py9tpF92iPuunowDxt6d/Hf/2d+dzaemjaY+Fmfl9gNMLMpnUN7Rx4/H4gn21NYTDhrDBuQ0lR+qj/HQss1kh4Lc9onTMTPKDxzh+Xd2kB92XDm5kLxAHGL1fP+Z1YwflsO0kbkUDwzywZ4DDM2FkIuSG4hTtqWCHVXVvLOtkrsvKSGLmBcu9UfDJh49tsx7rz50mByLkWOxVsu86VSy4NFQaAqIcBtlnV9elwhS0+AYUZDfwfqtj7RykkEVyj46Hcw+qSOpLVWHKCns1+KXCWPxBAcjMQZ7j6J3zrGzuo7RBXkp+6fsrJq6KANzT/76XTzheOyNbVx33hiyQ6kbPKIgSKG/bt7L9Y+sAJLflg9GoixcuolLJ4/g3OLBfOzHr7J9Xzvn9KXbfPXiM9i27zBPv72zS/sp7JfFvsNHL+QPHZBNVW3Hf5w/PX1UU7254SA/nz+VP67eycfOGEpOOMhbH+zn8RXJIJw7pYhn11QAcG5xAX/bdqDV3hwv3DaLSOQIe6trufv3KwlbjGyiZBHjszOKuG76cBa9tYXd+w7ypQtOoT5SR32kjlXbdrN5136+dMEpHDpcx4CwI1IfIYsY0YYIS9bsYO7EQqpqaolEIpQMCicDK97A4bo6sgNxQt7RVSyaDCwXa6BfKEEi2kBeKI7Fo1iqjqIAZ8GmUHChbKobjFBWLhbKJje3H4cTIYLhbFZX1BHKyuX0kYW8uLGaesJNr1svmsgPl26jgRDfvGoamw9EeXJVJeW1Cb4+5yzOHFkAgTAxC9LgguTl5DS7ftV4ijHcdH1qe00DtQ1GIBBkZ3UdF08cDiQHl3zxt2W8cMeFvL5lLzV1UT46bigj8nPIzQpSeTDCjO+/DMDnP1LM4jW7ePHOj7Fxdy3nFhcQCh49Oqs+0sDeQw1khwKcMrhlWP3khY088OpmAJ79ygXk54RJOEdOOMjftu1n+pgCRg7K5bVNeyktLiAn3HFYRKJxwsEAoWBAQZBK9z//Hiu37+f3X/5Im8t37D9CbSTGaUP7Mf6e5OOvL588gufX9pzTStL3DcoLU33kxO6Qv2zSiOOe/gyQIEys2StOlsValGUTZcElp/HAS+sYmgv1dUfIIkq2RRmaAyP6GxV7q8nyQi6bBrKIkW3R5HqNLzs63bh9FjGyaWgKx4B1z9+fuDNihLBQmAYXpC6enI+6IFFCxAgSI0iUIDFvPuqCXnnIK0++EgQYU9gfswCnDc/n+bV7SBAgToDRBf0gEGDrvjocAeIYCQLJlzMSGHECLZY5ryzRbN2xwwaQHQ6zeudB4i7AqUP6s3VvXdN6v/zBdxUEmXL9I2/y1837WPTlWXzxt2UcOBLla5eeyY9f2MhHxhbyn/8wg+t/vYJPTR/Fqg8P8GRZeaabLNKLuGQQNQuPrGbhESZGiDghixMmTsibT07HCZu3nJbLs9oob9wmua82llu8zX2FiBMwh+G8SHCNf+abTTsC3nIjQZAEwRQHnH3noIIgU440xFi6oZKrzh5JJBonnnD0y+74vGjjby0vvu0CbvjNCqqPRHnw+ukMz89mYG6Y/Jxw0yFoV33sjKH8+f2qY8ovnTScF9btSUkdInKy2g6PZGAcXdYYHgEcATu6fvP1Xrz/VgVBbxKNJzhSH2dgXhjnHA3xxDEXjSLROK9v2cu2vUcYOSiXSSPzKRqYg5mxq7qO368s585PjqOiJsKAnGTw5IaD/PaN7VxxdhHVR6KMHJRLbjjIL17dzJVnj+RLj5Xx2BfOY3j+0Yusz66pYPv+w+w71EDRwBxunHUqz71bQempg/noj149pu2tRzL1BrdfNI6fv7wp080Q6Vbbf3iFgkBSr/HI5f37Luf7z21g3PD+XHfuGPbURuifHWJATpgDhxv45E//zCv/PJuBeeEWRzuP/GUrf1y9iwevn86cs47+dsSRhhi7ayKMGZyHmTH27uf47tWTuHFWMfWxONmhIOUHjnCkIc4l3nOgHr7xHG5+bGWH7f3L1z/O8Pwclqzbzbvl1cw75xTGDu1HKBigoqaOgrwsrnrgNd7fc4hbZ4/lwWVH7zi//aJxfHz8MK75xV8BWHH3RRxpiPPxnyw74X+3/JwQByOdvwcjLyvIlNEDeXPrfj5TOpony8qZWJTP+oqDJ1x3d2p+ITxdLho/jJffq+zSPgIG3XQvaY+iIJBuUbzgWQZkh3j3O5d2epvNlYcYkBNqcdTRFTVHolgA8nOOhgzA+n+9lHAwQDgYYGd1HTsP1DGjZPAJ73/cN5+jf3aIVd+6pKn9w/OzGZCTHB7YvM5/vKCEy88qYkvlISaNym8aFvvLG87hsskjgOSRXE44yNL1e2iIJyjIy+K3b2zj8x8ppmRIv2NO+f3D+SV87dIzyc1qe2RI4wiTxvs9rnrgNdaU1/Dvnz+Xm/7jbwCs+86lvL+nllGDcqmPJRhdkMv2fUfYceAI55UUctm/LeeuORP44m9b/r/60bVTyM8NcdnkIn7x6mZ+/MLGFsvXfedSDtfHiCUc1UeiTByZz6oPD3DvM+t4p52fhL1h5hj2H27guXePvSC9/l8vJScU5IV1u5l+agHnef8W37piIrWRGNfPHMMDr2zmP17fxtKvXsieg/Wcf/oQNlQc5PKf/4UffPosrjx7JBXVddz4m7e4ZfZYvv3Muqb9N4Zoax/8YE6LYeGfnXUql00awaod1fz4hY3cMnssX7rwNAblZVEbifL02zu5bsYY9hyM4Fzyd08AfvrS+yxsdWT5zD+dz1UPJL88vH3PxUz/7kstlr++4BN85P5XWpRdMaWIhfOnsac2Qm44yKC8LGqORPnj6p08tGwLuw8m7/FZeN00Zp1WyNAB2cTiCR57cztXnT2SaNzx4vrdfOtP67hh5hjuu+YsQMNHpZscONxAKGhNfxQzrSGWYO7Cv7DkjgsJBlLzGIr6WHKoZHvjudftqmHuwteYUTyYJ788q8WymrooeVlBwsHO39S1pryaqx74K2/edRFD+me1GHLYGbF4glgiOdRw2cZKThmcx9ih/Tu17aKV5U13tN93zWRumHlq07KGWILFa3bxqWmjmv5otr7ZsNFDy7bwwyXvAclrTT+69mwG5oVJJByBgBGJxnnirQ+JJxxD+mdzx3+v5ufzp3L11FEt9vPm1n2MG9afwv7ZJ/Rv0CgWT/Dph15n2IAclm7Yw2vf+DhD+mezufIQpw/rTzBgROMJ8rJCRKJxXnmvko+OG9Llz/OGioP85IWN/PqzpQQCRiyeoOpQPUUDc9lzMMJTb5fzoyUbm74gnH73cy0eb/OLv5/O3Clt/7oiJJ+CcDASbXH/TFuef7eCj48f1jS8VEEg0k2cczy8fCufmjaKYSk6yskU5xyvbqxk9hnDWtys1dov/7yFcDDAFy4oaXN5POFYtrGST4wfhnXiuVCVByPd+m/nnCMSTbR7VJVpDy7bzI+WbOS+ayZz4bihTUcYqaYgEBHpwRpPGXan7g4CPcxERKQLujsE0kFBICLicwoCERGfUxCIiPicgkBExOcUBCIiPqcgEBHxOQWBiIjPKQhERHxOQSAi4nMKAhERn1MQiIj4nIJARMTnFAQiIj6nIBAR8TkFgYiIzykIRER8TkEgIuJzCgIREZ9TEIiI+NxJB4GZnWJmr5rZBjNbZ2a3p7JhIiKSHqEubBsD/tk597aZDQBWmtlLzrn1KWqbiIikwUkfETjnKpxzb3vTtcAGYFSqGiYiIumRkmsEZlYMTANWtLHsZjMrM7OyqqqqVFQnIiIp1OUgMLP+wFPAHc65g62XO+ceds6VOudKhw4d2tXqREQkxboUBGYWJhkCjzvnnk5Nk0REJJ26MmrIgN8AG5xzP01dk0REJJ26ckRwPnAj8AkzW+295qSoXSIikiYnPXzUOfcaYClsi4iIZIDuLBYR8TkFgYiIzykIRER8TkEgIuJzCgIREZ9TEIiI+JyCQETE5xQEIiI+pyAQEfE5BYGIiM8pCEREfE5BICLicwoCERGfUxCIiPicgkBExOcUBCIiPqcgEBHxOQWBiIjPKQhERHxOQSAi4nMKAhERn1MQiIj4nIJARMTnFAQiIj6nIBAR8TkFgYiIzykIRER8TkEgIuJzCgIREZ9TEIiI+FyXgsDMLjOzjWa22cwWpKpRIiKSPicdBGYWBH4BXA5MBK4zs4mpapiIiKRHV44IZgCbnXNbnXMNwO+Aq1PTLBERSZeuBMEoYEez+XKvTEREepFQF7a1NsrcMSuZ3Qzc7M3Wm9naLtTZkw0B9ma6Ed1I/eu9+nLfoG/3r7Fvp3ZnJV0JgnLglGbzo4FdrVdyzj0MPAxgZmXOudIu1Nlj9eW+gfrXm/XlvkHf7l+6+taVU0N/A8aZWYmZZQHzgWdS0ywREUmXkz4icM7FzOyfgBeAIPCoc25dylomIiJp0ZVTQzjnngOeO4FNHu5KfT1cX+4bqH+9WV/uG/Tt/qWlb+bcMdd3RUTER/SICRERn0tLEPT0R1GY2aNmVtl8aKuZDTazl8xsk/de4JWbmS30+rLGzKY32+Zz3vqbzOxzzcrPMbN3vW0Wmpl1VEeK+3aKmb1qZhvMbJ2Z3d7H+pdjZm+Z2Tte/77jlZeY2Qqv7v/2BjRgZtne/GZveXGzfd3llW80s0ublbf5+W2vjm7oY9DMVpnZ4j7Yt23eZ2e1mZV5ZX3lsznIzBaZ2Xve/79ZPbZvzrlufZG8kLwFOA3IAt4BJnZ3vSfYxguB6cDaZmU/AhZ40wuAH3rTc4DnSd5HMRNY4ZUPBrZ67wXedIG37C1glrfN88DlHdWR4r4VAdO96QHA+yQfCdJX+mdAf286DKzw2v0kMN8r/yVwizd9K/BLb3o+8N/e9ETvs5kNlHif2WBHn9/26uiGPn4V+C9gcUf19tK+bQOGtCrrK5/N/wT+0ZvOAgb11L51yx/WVv8Ys4AXms3fBdzV3fWeRDuLaRkEG4Eib7oI2OhN/wq4rvV6wHXAr5qV/8orKwLea1betF57dXRzP/8EXNwX+wfkAW8D55G8CSfU+jNIcpTbLG865K1nrT+Xjeu19/n1tmmzjhT3aTTwMvAJYHFH9fa2vnn73saxQdDrP5tAPvAB3nXYnt63dJwa6q2PohjunKsA8N6HeeXt9aej8vI2yjuqo1t4pwqmkfzW3Gf65506WQ1UAi+R/JZb7ZyLtdGmpn54y2uAQk6834Ud1JFK/wZ8HUh48x3V29v6BsmnEbxoZist+RQC6BufzdOAKuDfvdN6j5hZvw7qzWjf0hEEnXoURS/SXn9OtDytzKw/8BRwh3PuYEertlHWo/vnnIs756aS/PY8A5jQQZtS1b9u77eZXQFUOudWNi/uoN5e07dmznfOTSf5FOP/ZWYXdrBuT+5HayGSp5sfcs5NAw6TPE3Tnoz2LR1B0KlHUfRAe8ysCMB7r/TK2+tPR+Wj2yjvqI6UMrMwyRB43Dn39HHq7nX9a+ScqwaWkTzHOsjMGu+Tad6mpn54ywcC+znxfu/toI5UOR+4ysy2kXy67ydIHiH0hb4B4Jzb5b1XAn8gGeR94bNZDpQ751Z484tIBkOP7Fs6gqC3PoriGaDxCv3nSJ5bbyxCdIQPAAABdklEQVT/rHeVfyZQ4x1+vQBcYmYF3lX6S0ieV60Aas1spndV/7Ot9tVWHSnj1fkbYINz7qd9sH9DzWyQN50LfBLYALwKzGunf41tmge84pInU58B5lty5E0JMI7kxbg2P7/eNu3VkRLOubucc6Odc8Veva84567vC30DMLN+ZjagcZrkZ2otfeCz6ZzbDewwszO9oouA9T22b6m++NPOhZM5JEerbAG+mY46T7B9TwAVQJRk0n6B5HnSl4FN3vtgb10j+YM8W4B3gdJm+/kHYLP3uqlZeSnJD/gW4AGO3sjXZh0p7tsFJA8Z1wCrvdecPtS/KcAqr39rgW955aeR/GO3Gfg9kO2V53jzm73lpzXb1ze9PmzEG4HR0ee3vTq66TM6m6OjhvpE37w63vFe6xrr70OfzalAmffZ/CPJUT89sm+6s1hExOd0Z7GIiM8pCEREfE5BICLicwoCERGfUxCIiPicgkBExOcUBCIiPqcgEBHxuf8PBhC/zpHYksMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = language_model_learner(data_lm_full, drop_mult=0, qrnn=True, callback_fns=ShowGraph)\n",
    "learn.fit_one_cycle(10, 2e-3, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 7:14:46\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      3.496704    3.712812    0.324994  (1:26:51)\n",
      "2      3.444194    3.750016    0.320031  (1:26:56)\n",
      "3      3.539182    3.681418    0.327540  (1:26:56)\n",
      "4      3.480575    3.594445    0.340602  (1:27:00)\n",
      "5      3.575246    3.552679    0.348032  (1:26:59)\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD8CAYAAACYebj1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VPW9//HXZ5LJHpIAYUd2RVAWjSyiiBuCG23VFut1q5Zbl1qt7a3VutbWpbf2Xmuv1q1VS21/RatYd4WKWjdQQVZBBQ1rWEPInnx/f8yZk5nJTDKBLJC+n49HHjNz5jtnvt+cmfM+53vO+Y455xAREQEIdHQFRERk/6FQEBERn0JBRER8CgUREfEpFERExKdQEBERn0JBRER8CgUREfEpFERExJfaYW+clefGjBjWUW8vInJAWrRo0VbnXGFbzb/DQiGY35OFCxd21NuLiByQzGxdW86/w7qPHBpzSURkf9NhewrKhANMXS3UVkJtlXcb+VeV+Lamoonn48wPg/QukNGlidu8xtPTssGso/9LIge8jgsFaZn6eqhrYuVbWwk18Va08VbEzazI483L1e1b/QNBSM2A1PT4t2k5kF0Irh4qS2Hnl6Hbql1QtTs0vSmWAum5DSHRZLDkxZ+elgMBnXsh/94UCh3t/Ydg1YvNr6TrqvbxjSz+yjiY0fA4PTfOSjvyL8EKPTUdgpkJXpsOKemQsg8fNeeguswLidJQSIQDIzwt3m3peqhc0fC42WBLZi8lYm8lMoQiw0jB0mZqamooLi6msrKyo6vS5jIyMujXrx/BYLBd31eh0NGqSqFyZ2gFmtGldVbC/m1EmZTggdu9YhZaAafnAn33bh7OQU15THgkCpXd3v1dULYJtn7a8Fx9TfPvlRYnLJLdWwnfBlL2rp2dXHFxMbm5uQwcOBA7UD/PSXDOsW3bNoqLixk0aFC7vneHhYIOKXiOvTb010lV1tTxyFtfMGvyYIIpjbegf/S3xRzcM4dZk4e0bUXMQscd0rKB3ns3D+dCe21JBYsXKlWlsKcEtn/WML2uuvn3SssJBUdmV8gq8G67NnFbABn5nX4vpbKyslMHQm19PeVVdXTJDNKtWzdKSkravQ6dek9h7uINBAxOH9Wn2bI3P7uU5z/ZxMKfndQONet4xTvK6Z2XSUqg6S/XD//6MU9/tJ5XrpnMwT1zk56/c46aOsd989Zw3/w1dM1O49xxBzUqN2dRMcA+hcLDb37OxCHdGNknb6/nkRSz0B5aMBNye+79fGqroLKUZ95dzgcr1/GLUw+KHyyVO6F8O1Rsh83LQrcVOxIfX7FAKBjihkZ+4kAJZu59WzpAZwyEypo60lIDrNtWzp6qWrpkBOmVl+E/P+O+t9iwq5IPbmj79VOnDoWrnvwIaAiFiuo6HI6stMbNfuyd6FN/d5XXkJJi5KTv3b+oqraOuvr477Wv6usdyzeWcljflq0EK6rrWLphFwVZQU66ZwHfPXYQN5w2wn/eOcdHX+1kbP98/4v39EfrAZj6mwWsvfO0pN/rrpdW8cAbn/lBUFFd579HXb0jNc5eQyJ19Y7fzlvNxUcPIi+rcf/q7c+vAGhR/drLxl0V3PTsMh66oKhhYmo6r35Zz9WvlQHd+MXgKcnPsL4+tHdS7gVEODTCt5HTSjeEwqR8O9TsSTzP1MyIkGibvZIlxTvpmp1Gv4Ks5Nu6n1q+oZRuOWn07JKRsExZZQ2fb93D8F5dKK+uJT8rLWHZmtp6Pt28m6y0VKpqQ9+T0soaauoawn9x8a7Wa0AzOjQUnHPtmvqH3vQSkNzKY/Rtr5CTnsrSW09ptuxbq7fSo0s65dV1DO+VS0YwhRn3vc3KTbv99/rwyx04B0cOKPAfj+mXTyBgfLF1D/NXbuE7xyTuO1y1aTe7K2vYsruK4h3l/PKFlfztexM5amDXhK+prq2n3jkyginsLK9mzG2vAtC/a2jL8KE3v+CG00awduse8rOCvLp8Mz+eswSAq04YyqmjortZ6usdd7+8iqtPGkZGMNTn/eW2cn727FLuP+8IstNTqa2r58dzljBv5RYAdleG+uBv+8dybvvHck4c3oPXV27hmSsmcXiSofbGp1v4n9dWs25bOb/51pio5yJ/Y7yqto701Ia++Oraet5aU8IJw1u+Vb9jTzWXPr6QSUO6cU5Rf/p3bViZ1dU7dlfWkJcZZPWWsib3oCbeMQ+A381fwxXHDwVgS2kl3318Ly/cDARCK+XMgha97JDr/k4ee3j/mrGNgyQ2YPZ5r6Sg0fRZ933IDnJZdefXG83q5meX0js/k+8d18ZdiC3gnOOLrXsozE0nN6NhQ2Tnzp3M/sPDfOvCS6NCobSihpLdVQwuzKa6tp7Pt4ZCeOWmUgDOPWsGTz75JPn5+Y3eq7wmFATl1bXRT8RZNdbXt33HeweHQvPHPs97+F12V9Yy98pjqKt3zHzwHe49dyy98xrv8i7fUMrpv32TN39yAn3zo5/fXNpwtkJZVW3CPYDH31nLEQcV+OU2l1Yy+9113DtvDV/ccSq/fGEFxwwr5HtPLKJoYAFPXDKe/3jkvah5rL3zNFZu2h017Rv/9y8A/nzpeJ76cD1PfVjMMUO786dLx3POA++wtayKb48/yF/Zxjrlfxb494MpoX/aOQ+8w4UTB3DrjMOiyq7fWcG8FZu58dllAKz8+TROuqfh9V9tr4gqP+W//0mP3HTOHN3QzXbvvDXcO29NVLmH3/qcB974jCXFO5l96XgOu/ll9nh7AK8u38zXxvZl6A0vRr2mqjZ6pfK6FxZf+93bXDalYSVQV+94Zdkmph3Wy99QKK+uZcRNL/ONI0IHl//+0Xp+860x7K6sYXNpFc45LnmsYeV6yM9e4rUfHsdJ97xB1+w0Tju8N0+8u45nrpjEmP4NX8Zbn1tGzy4ZTa6Exv48FKCL1u3g3nlrojYkbn1uGY+/s46fnXYotz+/ghlj+jCkMIdvHdU/4dbjr15e5YdCeXX0WVD19Y5ARDfePa+s4pBeXeiWk0ZGMCWq7smqratn+cZSRvULvbaKNLaQxlfBQayt7cGxIxKPkvDp5t1M/c0C/vSdozimfzDuXsnar76iILCHPLcbKrZTvaOY7asX0T1QRmpdRaN5vuv9W9ztl7GxJpOu3XuR0aU7ZHZl2JLd7CCXmpQigjnd2R3IpSI1jx49++zTsZKNuypISwnQLSe92bJbdldiGIW56f7ebFlVLXuqajm8X8P/f8eOHfz18Uf41oWXRr3+sy2lpKSk4IBVm3cT64m/PUN+gnqkpyZomwt9Npaub9hLmPXEombbsq86NhRiHr+ybBNby6r59viDuP+fn9GvIJO312wDYOB1z/ONsX35YO0OJt4xj7V3nsYf3v6Ck0f09HdJZ7+3jnoH81Zu4fwJA/z5VlTXMf6Xr/uPq2rqEobCTd6KNCzydV9uL+ehN7/goTe/AODN1VvjzuOfq7YkbPO3H24IkLfWbKWsqpad5aEDj87B9j3VdM1Oo7SyhoAZ2Wkp/OuzbVHzqKlr+M899s46bj5jJB99tZO6ese4QV2ZdOe8qPLDb3wpYX2e8vr0t+yuoraZrZBfvrASgH99to1BP30h6rnnP9lIj9zGH/pXl29OOL/7//mZf3/y3fNZv7OCn0wb7odFOFif/nB91OuOvmMeu6titqo8Ly3dCIT+j0+8u86rwybG9M9n065KJtzRsDwvmDiAz0v2cFjfPJ75aD2vLN/E/513ZNz5DrzueVb+fBqrN5fxuNfV+Paa0PJ/9uMNALy4dBMrNpbSryCT+xPMB0IhHGnOomJKK2u49NjBAI3CONGe7ahbXmbsQQU89p1xQGiLOy8zyA+nHsKdL67k4be+4JVrJnP3Syv91xx79/yoeZZX1zLtf97kV2ePYvzgbixat50lXlfFi8s2c8zBh/Pimkoum72SRy8q4oTRob2uKc8+78+nrKqW++at4YGvQssznWqO7pPCho3ruXfGAIbkVHPjkwsooIxDs2uprCwhf0sZJ2dV4TYvY3rKJvIpI+W1ZwDI9f7C6glg6TlYMBN37P1UbagnLZiCWYB6AmCGWUpoY8IC3p9hZdVUYWBZYAHqnBEIpGCBABAqW+uMsuo6tu6qoh6jMCeN1VvKqPS23sPfCOcc5dV1XPaDaylet5ZvnnIsuVkZ5OTk0Kt3b95f+CF/n/cuX//a11n9xVqqqqo47zv/ydnnXQTA2BEH86d/zKN8zx6uuOAcTjjuWN555x169+nDE3+ZE3f5VtTUsXlXJd994i1/2msrEn+fWkuHhkK9c6RE7COFU/Db4w/irogPcli4fxtCu/e3PrecJ95Zxws/OJad5TXMfu9LAG58Zik3PrPULxvuNmp439DWdE56KqNvfYXvTErulK/dlfFXRLEu+sMHUY+/10S6H3bzy/795Rt3cdb9oa3/8DGOowYW8MHaHU2+3+DrX2jy+aZc+7fF/v0NOxtv4SXr1eWbmwyA5qz33vuul1ZyTlE/uuek8+7n2xqVe2nppoSBAPDfr3zaaNrv5n/Gj08ZzoqNpVHTR9wU+t+fengvXvhkExBa+WcE42+5xYbr/FXRZ4aE51+8o4Iz7nuLWK8u38whcbqa/uupUJdddV09XeP0PYf3bDeXVpKfFaRkdxVvfFpCaWUtb3xawm3PLeecon7+Z+aHUw/h4be+8OpSzmsrGm+kvLk6VPf01BS+3F7Otx58l2eumMRZ97/jl5n93pd8s6g/l83+EIDv/DG0V3bxpIF+ma+2l/tBE1ZFGvM3ABzEKc84bj5jNE/WeV0wEYt07SWnUVFdy5E3vYxRTy7lTOxtbN60kXzbTQFlFFgZ+bab6QOzGdY1lYr6VCpdkPo6uHtBCUs3VxJedTcstb3vYhlRGOTGY/NwFqAew2G4zRuoqHWA8d/XX8E3P13Kv155in+9+wHnXPBdFv/rNXK798axk1/d8XOy8/Ior6ji9NPO4OunnkReQTecqyeDamqp4csvPmP6fb/nmlvu5oeXXcIfZv+F07/xrb2uc2vr0FDYVlZNr7wMXvhkI5d7Hzwg6gBLpLzMILsqQn3UZd6K4fOte5rcEo7nP59YyIdf7vQfP/r2F0m97vTfNv6iD7zu+SZf45zjpWWbkpr/f3n9+ZEHvZsLhNb0yj6s1FtT0e2vseK2aTy/ZGOj5773p73bfQ4f+I8nHAhhlTXNXD3tifw8JqO54wh3v7Qq7vTDbn6Zedcexwm/foOTDu3Bqs27o7oAH337C8YNaji2FNndsPir+Acoz3/kfQD+cNFR/rSz7v9Xo3Izfvd2o2l/eHutfz82EOKJ3COMVFNX7y8TR4BScnh5I8DQRut1638wQ44fyvZ3P2Kd6wG1UJ9RTUXEMY+s9FSIOE65p6rW2+QMzSx8P5gSoNZbxxgutNPgPeeCWWwlD8MR8P6sNny/nnBUZFBNsL6Ko0aPZEhhNridmMEDjzzA318M/U82bdxI1dpFDO42ilTqGBLYSFmggkH9+3DW4XnAOk4YdRDVxUsYaRP8EIq8rbNdPBz8FZWkUUUaFS6N85v9j++bZkPBzDKABUC6V36Oc+7mmDIXAb8Cwpvy9znnHm5u3hPueJ2/zJoQFQgAw2L6pcPGDerqb40m82FMJDIQ2tqjEV+g5nxW0sQZIv9mtu2pYtmG0uYLJumQn7VswyEZLQmEfXXCr98AiLvVD6Gu07DIjZf/fX11k/N95K2GDaJEobmvtuyOfzX+8Btf4k+XjE9qHr9+9VNOGhF9wsDVJx3MV9vL/cdGaPU/ql8+zjk+Wd/yM3YCZmxyif8P610NVS6VT10/vmQtLqsbn9SHuqoX/ust5r65mEeffZ3szEwuOGcGaypzKazvTS0pfFnfg4r6MgLpmax33TAc5YFsymv3sJ2chhDyb0Px0Mt2kEE1GVZNOklc47KPktlTqAJOcM6VmVkQeMvMXnTOvRtT7q/OuStbWoGZD8bOJrF96Z7oKD//x/KOrsIB6Zi79j70/x0lOr7VnLfW7N3rWkNdvePch5L//k//3zd56MyGM+IiAwGidy7qm1ixN6W512Xn5FC+pyziPRvK7y4tJTcvn2BmLqvWfMrijxZRRRp7yMBh7CaTcuqpcwG2uS4AlLnQtI2uW9z3K3EVfLf6lzFTT9+rtiWr2VBwoXP+wv+FoPenC5JFZL8UPvOqLeQXdGVM0Xi+ceJEMjIy6VrYcBbXpCkn8rc/PcrZJ09i4JBhjBpb1MSc9l9JHVMwsxRgETAU+J1z7r04xc4ys8nAp8A1zrmv4sxnFjALIK3X0L2utIhIIm3drXfnffF7xtPS0/m/J+KfSfTiO6HjhQVdu/H06w0H8y/83vdbv4L7KKmTf51zdc65MUA/YJyZHRZT5DlgoHNuFPAa8FiC+TzonCtyzh2YEdqJjO6X+MKxS5q4iE5kf7d+H86ikxb+8ppzbifwT2BazPRtzrnw0aSHgMQnact+4b5vH5HwuRtOPZSPbzqZY4Z296eFr8SWxp678piox7d/LXabSf7dFMa5ZudA0WwomFmhmeV79zOBk4CVMWUix0M4E1jRmpXc3115/FCevvzoDnnvtXee1uSwHb8+Z3Tc6f27ZrH2ztP49PbpXDhxQNRzgYCRn5VGv4KGq8JnTR7conpdEDNPgLd+cjxrfjG9RfPpSBMHxz/4F+vwmL2u/5gwgPeuP7EtqrTfevEHx3Z0FfYLBVlp9O+aFXfEhQNFMnsKvYH5ZrYE+AB41Tn3DzO7zczO9MpcZWbLzGwxcBVwUWtXNNlxciLde+5YrjpxWNS08JAJrelHpxziD42RyI9POSTu9EuOGcRFRw9M6n3uPy/x1n0i6cEAf/veRP9xbsyV3GmpgUbDZIRFXuEcNUzEmSOjyhXmpkfV7R/fPybuyLR98jJbNBDe3ops7754/JJxfHHHqU0un3A33O/PP5L7zzuCZd5YWbHDXSQK7j9cdBQnDu/RKvUFooYq2Vvv39DyQAsPvbK/aW4UYIDuOekJh5eJlHA4Cs+Qwhz6d82iwLsA8aCuWQzrkZNcRaHR0DyRTjp0H0blbaFmv6HOuSXOubHOuVHOucOcc7d5029yzs317v/UOTfSOTfaOXe8c67x5cj7aMLgxAO/hX32y1P9+//80RTOHN2H75/QcEB78U1T+fU5o/1hAfbVg+cfGbWH0NQX8orjhzLv2uN496fRX7gbTx/B0CQ+OD8+5RCmjuzF2jtP4+ELipg4uBv3njvWfz7T+1DHG0sqHFiDumfz0jWTeeTCxId0RvTu4t+PDIKeXTL4x/eP4b5vj+XCiJXk4pun8saPpzD98IadxcP65jGgW2joke8e23B8IpDEFzRZc6+clPBLNLh7dtTju88axfTDevmP/57EXt1fZ00gmBLAzLjlzJH+68PzLsxN5/fnH8mzXtfRKSN7Mf3w3mQ3Mapu5HLulp3G+zecyPHDe/DIRUex4raGHtnc9FR6dtm77oeDujYehfThC4pY3cwe2ivXTAZCK6YeuYlH/wy75YyG0XUDRrOjn86+NPH1CNeefHDc6U2tJJPV1OikYX3yM+ma3XS5/gVZzW7QxC77/Kw0MtNSGRCxTCK/XwO7Z0eFRrec9NCGUyBASsQXefHNU3m4ie9sa+uwX+QY2K3hH9U9ZqCo3nmNP5RpzaQ0RG8VhBdQasS0vKwgZsZxBxdGTY/150vH8/MZoa3he745mqy0FAIGv/j6YVw2ZQh3fONwumWncfKInlF7CCfEbPE9/p1x3DZjJCt/HvrCDy7MidvXePaR/fz7Jx0aPY+stBRevWYyVxw/1G/fSSN68uSsCVEhtOLn01h752l8ccdpUVu2Rw4oICVgrL3zNOb/aAp98zM5Mc5Wx+pfTOeeb47mjxc3XOF67riDuHzKED688WQgtLKP3QPIywz6w4OfMbqPv+Xcs0sGS26ZyvWnHtrovRJ56eroLojBhdn8+buhlckPY1YchvHPH0/xHw/v1TB8RDDisxJMMb55VP+o0WQjVxThlduNpzes5KDxOdf/O3Msi352EmO95X3zGSM4ZWQvWiLyiz73+8dErXwz0xq2VN+9/kRaeh1ZuBsxNngHdc/mpBE9o37g6JkrJjV6/cE9c/nzpeN5/drjAHj68qN58QfHcvfZowDonhO90rxo0iAuPWYQEwd34/M7TiMjmMK9546N+/k+amABkyKOT8UKr2z/c/JgfhdxrCu8fDOT2IoHokayDQtv3acEjEATo292yWj6RMyMYIDsJobBH1yYeMMuLyuNQm9vJDJYumQEyYyZZ/fcdEb06cLIiJ6RvMx/k5/jjFxAc6+cxNHeIG7De+UyY0zfqLGPjh3WnQuPHsg7n23j7CP7c/3fPwFgdP98Fn8VfXVyt+w0tu2pJjs9vOUc/4OQk5HKzvKGU9cePP9IXly6iWE9czh6aHeOHtqd8ycOBOAbR/Rr9Pp4PxjztbF9Oe7gQuYu3sA5Rf3ISktlMtGjUYa/s2mpAf7XGwY6I5jij7gZObrqHy8+ivGDukWtMJIxICJwk+3bDKYEGrUzJWD817Thccv/8eKj+DzmCuzfRuy5QOhDD/DoRUVRV+KuuG0aDuePPRRW6G0cpKcGuPvsUcwYE+rq++KOU/msZA/3vBo9rlHkiu6+b4/1R4INeiNqmuEPfR65jo28cveiSYO4yBv76txx/fnm799h6frSRiuAtNTQaJs/O+1QumYHWxwIAI99ZxwT7nidy6cMibsVPP9HUyitqCE7PZX/GD+A37zWeBynsMGF2ZRW1LK1rIr/i+i6C69Ajz+kkPmrSphySMPn79xx/fm8ZA9j+uez9NZT+OnTn/Dc4g382duKPzpixR3e2AlvjKWlBDhheA9/SHSAn8UE6Zmj+1BRXctPnvrEn/aDE4dFjZUU9ouvH8YNfw+NT5bjfVez0lI5bVRvThk5nYAZAW9jpqyqNmqMsHgKstLidu8UZAVJDRhZaak4HKu871dhTjrb9lT7F6ulpaYwql8+KzaWxh1mJzMtlYxgCllpKeyqqGFHecOVxQd1zSI7wXc0JyeHsrIyXPkOrr/qKubMmRMa/ypitfT9877GL+66C/o1fXV3dlpKizcW9kaHhUJWWioTDinkuumH0ic/kzW/mM69r6/mkmMHE0wxqmvr/S/Ff0wYQI/cDJ6+PLSF8+3xoRWyc456B0Ouf8E/U+Yvsybw2ootUT9us+SWqWwpjf6h75z06FCYOrIXU/fiix6rIDstqnsllpnx7k9PpCA7GDX2/8kjenL78yu46OiBXDxpEMbed7ecP2EAr63YzF1njdqr1ydjyiE9mBL/MEkjJwzvGfWbBuGQ+823RpOflcbF3gCC4WX2n8cN8QMBQv+z2C62VK8POyc9lbKqWnp4ffh5mUHSUwPkZQa5/tTh/v848oseeQA9UlZaqr81X1sff/yjguy0qB8mako46MN99L3yMpo8KWBQRLfXVScOZcHqEhat28EVxw/hd/Ojxw+ad+0U/vj2F9zy3HKOHdawMg9vAw3rmctVJw6L+jW6O77R8HnISU/ll18/jK+N6RMVBrHCn8BgaoBHLzqKJ9//0g/veL511EEs31DKY++sixrxNtLfvjeRw/vmccPflxIwmDnuIMqq6vzwiO2myUlPZdXt0wgGAixbvpwhPXJI8UIjPAhhry4ZBFMDDO/Vxf8NA4CUQCBqz3BEny6UV9fRJSMYd5jz3nkZfLm9nMGFOazbuoe6iCuczYwumUFyM1Lpk5/Jsg2hYTSS6qLq04c5c0LXMKTH7PmYQUZq41AZXJgDOxv+1x/dNDX0+tubfbt90mGhYAZ/uLihbz81JcAPpzasZX5w0jA/FLol6O8zM1Is9HsB4e6gYT1zGRYzEmWXjKC/1RoWubX4r+tO2LfGtFCvON1jA7pls/Ln00hPDezzDw+lpgSYfemEfZpHe/j62Og9k8y0FFb/YnqTXXsQ6vIJdxfdeuZIbnluGVnBlKgV7uKbp0a9JjwA2mVThpARTOG1Hx4XdzTU704ezJV//oiB3bIbPddSlx472B8Ou6XMjIcuKOKlpZv49viDokJhmrfxErmXExbuJjt6SDe/qyuR3Ixg3K7ESOEt6fAKNN4ecqwUb08t0cHnsf3zSU0JRC2veOERKRzuqSmBqA2+Xl0y2FRa6W8kpKUGOLR3l0Yj4oalBgJ0yQjVL95GV35WGtnpqQRTAuRmBtlZXk3/iOMlP/nJTxgwYACXX345WWmp/PrO2+mVl8mCBQvYsWMHNTU13H777cyYMSNqvmvXruX0009n6dKlVFRUcPHFF7N8+XIOPfRQKiriX1eR49UjLJku9NZwQPwcZ0EzB4GSOXMg1sZdDXsOfVrhgFZr2Jt2dAbvR/ShB5M4OylyRXjWkf0468jG3XuxThnZi/9+5VP/OEyig/unj+qT1G96t4eu2Wn+XnGkB85PfBnQkQMKWHHbtBZ3OSYypDCH608dHrcLNZHR/UN7J4l+ka7VzkB78Tp6bPqE2HO3UnEMrvJ+yKilP6fb63CC0+8EGvaSIntsZs6cydVXX83ll1/OkMJs3nhpLi+99BLXXHMNXbp0YevWrUyYMIEzzzwz4cbd/fffT1ZWFkuWLGHJkiUccUTLzypsSwdEKMT78RbpPHo08Vu3rWVYz9z98jeck/X8Vcfw3ufbm/zJ1rDWCgQI7bHMmtyyn8mcMaYvY/rnM6AV9rb2hsX7HctWMnbsWLZs2cKGDRsoKSmhoKCA3r17c80117BgwQICgQDr169n8+bN9OoVvzt6wYIFXHXVVQCMGjWKUaParpt3b+zXobDklqksW18a9RupreX2rx3GzyJ+iEf2fwVZwSbP8ujMRvbJizo+sL+LFwhPfncC81a24kjH3hZ9PJ8Xh05AGdWv5T9lGtYQLdFHd88++2zmzJnDpk2bmDlzJrNnz6akpIRFixYRDAYZOHAglZWVsbOLnnc7/jZ9S3XYKanJ6JIRZOKQ5K4qbanweefNnZ8s+4+PbprKU5d1zJXjsu8mDumW9EH6fTW0MCfqNOW9Ed7jSkuJ3vOaOXMmf/nLX5iXkR5OAAAQIElEQVQzZw5nn302u3btokePHgSDQebPn8+6devizc43efJkZs+eDcDSpUtZsmTJPtWzte3XewptKdx/H3nKnoh0DlktPZYQR9fs0EHn2GN9I0eOZPfu3fTt25fevXtz3nnnccYZZ1BUVMSYMWMYPjz+adxhl112GRdffDGjRo1izJgxjBvXOhfTthZze/ljFPuqqKjILVzY9M8TtrWvtpfTs0tGux3VF5F9s2LFCg49NPkLIg908dprZovacqTpf9s9BYh/BaSIyL8zbSKLiIhPoSAiB5SO6vJubx3VToWCiBwwMjIy2LZtW6cPBucc27ZtIyOj7a/hifVvfUxBRA4s/fr1o7i4mJKSko6uSpvLyMigX7/kryRvLQoFETlgBINBBg3Sb4i3JXUfiYiIT6EgIiI+hYKIiPiaDQUzyzCz981ssZktM7Nb45RJN7O/mtkaM3vPzAa2RWVFRKRtJbOnUAWc4JwbDYwBpplZ7C+4XALscM4NBX4D3NW61RQRkfbQbCi4kDLvYdD7iz1JeAbwmHd/DnCi7c9jw4qISFxJHVMwsxQz+xjYArzqnHsvpkhf4CsA51wtsAtomzGvRUSkzSQVCs65OufcGKAfMM7MDospEm+voNElh2Y2y8wWmtnCf4eLT0REDjQtOvvIObcT+CcwLeapYqA/gJmlAnnA9jivf9A5V+ScKyos1O8YiIjsb5I5+6jQzPK9+5nAScDKmGJzgQu9+2cD81xnH5xERKQTSmaYi97AY2aWQihE/p9z7h9mdhuw0Dk3F3gEeMLM1hDaQ5jZZjUWEZE202woOOeWAGPjTL8p4n4lcE7rVk1ERNqbrmgWERGfQkFERHwKBRER8SkURETEp1AQERGfQkFERHwKBRER8SkURETEp1AQERGfQkFERHwKBRER8SkURETEp1AQERGfQkFERHwKBRER8SkURETEp1AQERGfQkFERHwKBRER8SkURETE12womFl/M5tvZivMbJmZ/SBOmSlmtsvMPvb+bmqb6oqISFtKTaJMLXCtc+5DM8sFFpnZq8655THl3nTOnd76VRQRkfbS7J6Cc26jc+5D7/5uYAXQt60rJiIi7a9FxxTMbCAwFngvztMTzWyxmb1oZiMTvH6WmS00s4UlJSUtrqyIiLStpEPBzHKAp4CrnXOlMU9/CAxwzo0Gfgs8E28ezrkHnXNFzrmiwsLCva2ziIi0kaRCwcyChAJhtnPu6djnnXOlzrky7/4LQNDMurdqTUVEpM0lc/aRAY8AK5xz9yQo08srh5mN8+a7rTUrKiIibS+Zs48mAecDn5jZx96064GDAJxzDwBnA5eZWS1QAcx0zrk2qK+IiLShZkPBOfcWYM2UuQ+4r7UqJSIiHUNXNIuIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4mg0FM+tvZvPNbIWZLTOzH8QpY2Z2r5mtMbMlZnZE21RXRETaUmoSZWqBa51zH5pZLrDIzF51zi2PKDMdGOb9jQfu925FROQA0uyegnNuo3PuQ+/+bmAF0Dem2AzgcRfyLpBvZr1bvbYiItKmWnRMwcwGAmOB92Ke6gt8FfG4mMbBISIi+7mkQ8HMcoCngKudc6WxT8d5iYszj1lmttDMFpaUlLSspiIi0uaSCgUzCxIKhNnOuafjFCkG+kc87gdsiC3knHvQOVfknCsqLCzcm/qKiEgbSubsIwMeAVY45+5JUGwucIF3FtIEYJdzbmMr1lNERNpBMmcfTQLOBz4xs4+9adcDBwE45x4AXgBOBdYA5cDFrV9VERFpa82GgnPuLeIfM4gs44ArWqtSIiLSMXRFs4iI+BQKIiLiUyiIiIhPoSAiIj6FgoiI+BQKIiLiUyiIiIhPoSAiIj6FgoiI+BQKIiLiUyiIiIhPoSAiIj6FgoiI+BQKIiLiUyiIiIhPoSAiIj6FgoiI+BQKIiLiUyiIiIhPoSAiIr5mQ8HMHjWzLWa2NMHzU8xsl5l97P3d1PrVFBGR9pCaRJk/AvcBjzdR5k3n3OmtUiMREekwze4pOOcWANvboS4iItLBWuuYwkQzW2xmL5rZyESFzGyWmS00s4UlJSWt9NYiItJaWiMUPgQGOOdGA78FnklU0Dn3oHOuyDlXVFhY2ApvLSIirWmfQ8E5V+qcK/PuvwAEzaz7PtdMRETa3T6Hgpn1MjPz7o/z5rltX+crIiLtr9mzj8zsSWAK0N3MioGbgSCAc+4B4GzgMjOrBSqAmc4512Y1FhGRNtNsKDjnzm3m+fsInbIqIiIHOF3RLCIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJrNhTM7FEz22JmSxM8b2Z2r5mtMbMlZnZE61dTRETaQzJ7Cn8EpjXx/HRgmPc3C7h/36slIiIdodlQcM4tALY3UWQG8LgLeRfIN7PerVVBERFpP61xTKEv8FXE42JvmoiIHGBaIxQszjQXt6DZLDNbaGYLS0pKWuGtRUSkNbVGKBQD/SMe9wM2xCvonHvQOVfknCsqLCxshbcWEZHW1BqhMBe4wDsLaQKwyzm3sRXmKyIi7Sy1uQJm9iQwBehuZsXAzUAQwDn3APACcCqwBigHLm6ryoqISNtqNhScc+c287wDrmi1GomISIfRFc0iIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4kgoFM5tmZqvMbI2ZXRfn+YvMrMTMPvb+Lm39qoqISFtLba6AmaUAvwNOBoqBD8xsrnNueUzRvzrnrmyDOoqISDtJZk9hHLDGOfe5c64a+Aswo22rJSIiHSGZUOgLfBXxuNibFussM1tiZnPMrH+8GZnZLDNbaGYLS0pK9qK6IiLSlpIJBYszzcU8fg4Y6JwbBbwGPBZvRs65B51zRc65osLCwpbVVERE2lwyoVAMRG759wM2RBZwzm1zzlV5Dx8Cjmyd6omISHtKJhQ+AIaZ2SAzSwNmAnMjC5hZ74iHZwIrWq+KIiLSXpo9+8g5V2tmVwIvAynAo865ZWZ2G7DQOTcXuMrMzgRqge3ARW1YZxERaSPmXOzhgfZRVFTkFi5c2CHvLSJyoDKzRc65oraav65oFhERn0JBRER8CgUREfEpFERExKdQEBERn0JBRER8CgUREfEpFERExKdQEBERn0JBRER8CgUREfEpFERExKdQEBERn0JBRER8CgUREfEpFERExKdQEBERn0JBRER8CgUREfElFQpmNs3MVpnZGjO7Ls7z6Wb2V+/598xsYGtXVERE2l6zoWBmKcDvgOnACOBcMxsRU+wSYIdzbijwG+Cu1q6oiIi0vWT2FMYBa5xznzvnqoG/ADNiyswAHvPuzwFONDNrvWqKiEh7SCYU+gJfRTwu9qbFLeOcqwV2Ad1ao4IiItJ+UpMoE2+L3+1FGcxsFjDLe1hlZkuTeP8DVXdga0dXoo105raB2ncg68xtg1D7BrTlGyQTCsVA/4jH/YANCcoUm1kqkAdsj52Rc+5B4EEAM1vonCvam0ofCDpz+zpz20DtO5B15raB376BbfkeyXQffQAMM7NBZpYGzATmxpSZC1zo3T8bmOeca7SnICIi+7dm9xScc7VmdiXwMpACPOqcW2ZmtwELnXNzgUeAJ8xsDaE9hJltWWkREWkbyXQf4Zx7AXghZtpNEfcrgXNa+N4PtrD8gaYzt68ztw3UvgNZZ24btEP7TL08IiISpmEuRETE1yGh0NywGfsTM1trZp+Y2cdmttCb1tXMXjWz1d5tgTfdzOxer11LzOyIiPlc6JVfbWYXRkw/0pv/Gu+1bXrRn5k9amZbIk8Hbo/2JHqPdmjbLWa23lt+H5vZqRHP/dSr5yozOyVietzPp3eyxXteG/7qnXjRLsO8mFl/M5tvZivMbJmZ/cCb3lmWXaL2dZbll2Fm75vZYq99t+5tnVqr3Qk559r1j9DB6s+AwUAasBgY0d71aEF91wLdY6bdDVzn3b8OuMu7fyrwIqHrNiYA73nTuwKfe7cF3v0C77n3gYnea14EprdxeyYDRwBL27M9id6jHdp2C/CjOGVHeJ+9dGCQ95lMaerzCfw/YKZ3/wHgMu/+5cAD3v2ZwF/boG29gSO8+7nAp14bOsuyS9S+zrL8DMjx7geB97zl0qI6tWa7E9a1tRufxD9nIvByxOOfAj9t73q0oL5raRwKq4DeER/mVd793wPnxpYDzgV+HzH999603sDKiOlR5dqwTQOJXnG2eXsSvUc7tO0W4q9Uoj53hM6um5jo8+l9qbcCqbGf4/BrvfupXjlr42X4LHByZ1p2CdrX6ZYfkAV8CIxvaZ1as92J/jqi+yiZYTP2Jw54xcwWWeiKbICezrmNAN5tD296orY1Nb04zvT21h7tSfQe7eFKrwvl0Yiuj5a2rRuw04WGcYmcHjUv1w7DvHhdCWMJbW12umUX0z7oJMvPzFLM7GNgC/AqoS37ltapNdsdV0eEQlJDYuxHJjnnjiA0SuwVZja5ibKJ2tbS6fuLztCe+4EhwBhgI/Brb3prtq3d2m1mOcBTwNXOudKmiiao03697OK0r9MsP+dcnXNuDKFRIcYBh+5Fndp8uXZEKCQzbMZ+wzm3wbvdAvyd0MLcbGa9AbzbLV7xRG1ranq/ONPbW3u0J9F7tCnn3Gbvy1gPPERo+UHL27YVyLfQMC6R06PmZU0M87KvzCxIaIU52zn3tDe50yy7eO3rTMsvzDm3E/gnoWMKLa1Ta7Y7ro4IhWSGzdgvmFm2meWG7wNTgaVED+txIaH+T7zpF3hnfkwAdnm72y8DU82swNv9nUqoX28jsNvMJnhnelwQMa/21B7tSfQebSq8MvN8ndDyC9dnpneWxyBgGKEDrXE/ny7UITuf0DAusW1o82FevP/nI8AK59w9EU91imWXqH2daPkVmlm+dz8TOAlYsRd1as12x9fWB4wSHGg5ldDZBZ8BN3REHZKs52BCR/EXA8vCdSXUT/c6sNq77epNN0I/SPQZ8AlQFDGv7wBrvL+LI6YXEfqgfwbcR9sfoHyS0G54DaGti0vaoz2J3qMd2vaEV/cl3heqd0T5G7x6riLirK9En0/v8/C+1+a/Aene9Azv8Rrv+cFt0LZjCO32LwE+9v5O7UTLLlH7OsvyGwV85LVjKXDT3taptdqd6E9XNIuIiE9XNIuIiE+hICIiPoWCiIj4FAoiIuJTKIiIiE+hICIiPoWCiIj4FAoiIuL7/2+lMO8LfWTfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(5, 2e-3, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 1:28:47\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      3.489192    3.555320    0.348238  (1:28:46)\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8VuX9//HXJ4sQVhhhyApLNgJGxA0UlaFiK21Rv646WsfP8e3ChVtp7fBr69ZatbZVcdQKuFEcOIJCAAEBjRpACHsGMq7fH/e579wzuQMZkPN+Ph555NznXPc515Wc+3yudc5tzjlERMS/Uho6AyIi0rAUCEREfE6BQETE5xQIRER8ToFARMTnFAhERHxOgUBExOcUCEREfE6BQETE59Ia7MBZrdzQAX0a6vAiIgel+fPnb3DO5dTmPhssEKRndyA/P7+hDi8iclAys29qe5/qGhIR8bkGCwQOPexORORA0HAtAsUBEZEDQoONEYiI1FRpaSlFRUWUlJQ0dFbqXGZmJl26dCE9Pb3Oj6VAICIHjaKiIlq0aEFubi5m1tDZqTPOOTZu3EhRURE9evSo8+NpsFhEDholJSW0bdu2UQcBADOjbdu29dbyUSAQkYNKYw8CQfVZTgUCERGfa8DpoyIiB5ctW7Zw//331/h9EyZMYMuWLXG3lVVUUF7RsFdEtQhERJKUKBCUl5dX+b5Zs2aRnZ0dd9sXa7bx5brttZK/faVAICIHjZLScgqKtlBSWvWFt65MnTqVVatWMXToUI444ghGjx7NWWedxeDBg6mocJx++ukcfvjhDBw4kIcffjj0vtzcXDZs2EBhYSE9+/Tl7PMuYODAgZx00kmU7N5NaXlFg5QnSNNHxTeOmf42l4/uzVlHdmvorIgnv3ATkx+cx7M/P4oRPdpUmXb33nI27NhLhzawraSU3726jC/WbKvV/Aw4pCU3nTow4fbp06ezePFiFixYwDvvvMP4CRN5/s0PeXDYABav2cp9Dz5M547t2b17N3l5R/CD8afSq2snAErLK1i6dhvffr2K6X99lKefeJyf/OQnvDn7ZU750U9rtRw1pRaB+EJZeQWrt+zmuhcXNXRWQl4pWEPe7W/Uy7EausYZzTnHBY9/wuQH5wFw71srOPvRj8idOjNuXssrHP2nvRp6/f3WElycbvXS8gp27ilLKg8V8XYQtS8ItEJ27Y2/z0FDh9OlW3cKN+4E4M67/8xhhx3GyJEj+fa7b5n3+eJQfkr2Bloxnbt2p9/AwWzeuZfhw4ez5rvv4u57e0lpvf3fDroWQWl5Bemp/o1f87/ZxBkPzOORc/M4cUCHhs7OQWNbSXIXh321qngHb36xjkuO75nUtL9vN+7iin9+DsD67SX85/M1tG/ZhElDO0eke2vpOi58Ip8F004kOysjtP7ml5cwtGs2pw+LTB/P3a8t4745q5h37Rg6tWrK4tVbKatwDO0av8+6Pixdu505y4tDr1s2TWPWou8B6HP9bAqnTwRg0869DL/tDa4c0ztmH9eceCitmkbedVtQFBiQ7dAyk7QUIy01JSbNktVbKfeCQJ/2zUlLTSE9NYUtu/aypyxw4V2zZTcbduyhe5ssvtm0K5Q2aNPOvZSWV9A0Kyu07tN57/Phe3OYN28eW0uN08adyJ49e1hVvIPS8opQ4EnPCPwfv9u8iw27yigvj39ufr1hJ+u27any71hbGvSK+o0XRZO1qngHfa6fzR9fXx5aV1ZewfptDXu7+VfFO/bpfR+s3MBN/1kcd1vR5l1x17++ZB0Af317RY2P99S8Ql76fHXCWkZFhcNVU0uqLWXlFexIsuZWG7bs2lun+z/3sU+4a/Yytu1Orkzn//2T0PLWXaXcMWspV/17QUTNs6LCceETgUe1P/LeV6H1JaXl/P3DQq5+ZgEQ6DLZ7dU295SVc/ajH7Hgu8oZKvfNWQXAzIK1AJzyl/c5/b4PePHzIh56d1XCPO4pK2fSfR/w4aoNAMxbtbHKGuqioq3kTp0ZuhhXZU9ZZB9/MAhEm3Tf+wDc+/bKmG3Bz4hzjkVFWyOOu25bCau37I57jSkPO8dXrN/B0rXbKC2v4NtNu1i3rYQ9ZeVs2BG4AAeDQDBtRmYW27dvp2jzLr7dFPkZ3bFtGy1bZZOVlcUnny+i4PPIx+zHmxnkGni2UFC1gcDMMs3sEzNbaGZLzOyWOGnON7NiM1vg/VyUzMFPuPsddu0t46xHPkpq1Hzq8wUA/OXtlWzeuRfnHL2vn82IO9+q9mJcWl7BtpLSZLIVSn/di4tiLowlpeUs+76yX3LSX99nzB/f5fUl8U/kqpz96Mc8Me8bvos6of7x0Tcc+7s53PbKF9zw0iJyp87k242BNO1bZgIwrFvr0PFzp86kLOoD+uHKDaH3QOAkvPE/S7j6mQX0uX523Pz0vG4WPa6dxW9mLIzZ5pzj/RUbIv4eL32+ukYX89/OKCB36kxyp87kF/+Yz6CbXmPTzvgX6MNueZ3cqTNj1m/YsYetuxP/H1dv2R13fU2DzgWPf8Lz84tC773xpcWhi+23G3eRO3UmJ/95bsxxdyToQgB4fn5RqEx53VuH1q/dWlmR2RHWcvno642h5eDFHALdIkG5U2fSf9qr9J/2Kuu2lfCr5wr4YOVGTr/vAwBufKmyovGvT76NyM81zyzkrtnL2LqrlFcK1nDX7KWMu2cuuVNn0veG2XyzcRcLv9vCWY98zK3//YIzH/mIPtfP5p8ffxuRh6DXvM/Am0vXJ/wbBL23YkPMujbNKls8W3btZfPOvXy3Kf7/EyovrOUVrsqnGTvn2FNaTmlZ4iC2dG3lZ7p4e+Ja+IayDI46+mh+9IOj+PPt0yK2HTPqB1SUlzNw0GDu+8OdDBmWF7E9OvjFy2fwd/hguHOO8gqX8LNSG5LpGtoDjHHO7TCzdOB9M5vtnPsoKt0zzrkrapqBi57I58NVGznpz3O556dDq2zqflq4GYDubbMYdltk3+p5j3/Ce78ZAwQ+lB1aNCEtrAtp2K1vhC4Gb/7v8fRu3wKoHKxKSzFW3DE+1KwPXiznrdrInF+NCu2n342BfsqCm0+iZWY6C4u2AnDJU/PpnN2U6yb0Z+KQTjX6G7y3YkPEAGawFvPY+1+H1v1qxkKe/flRoRPk7x8Wkp1Vefy1W0vo2qaymXrWox8DUDh9Ih9/tZHvo1pNZeUVob+Pcy7ig/lsfhG/n3wY328tYeRdbzGgU0u6tmnKa0vW8euT+3LigA78/tVlvLl0PaP75vD4BSOSKucz+ZV9ocGLxR9eX86dPxwckzZ4sQ92BT4/v4i2zTM4//FPQ+WK9tj7X3PbK1+Eus3WbSth995ycts1Y+eeyg9WSWk5mempMe9fsW47XdtkYQZzlhczZ3kxJw/qyD1vfMlTH33DUx99Q+H0iRx/9xwAlq/bTvH2PRE1var6p3/5XCDArttWEtHNU7S58mK3raSM9i0Dy4nmlm9O0Lo58s636N+pZeh1dCBdVbyTH93/Qcz7znr0I5ZEDbruKauIaJX+7YPKczE4zlI4fSIbd+whLSWFVlnpLPs+UJnLyoj929792jLKK+C5/O849bBD+PuHhTFpwi90Q2+NP3bSsWWTiEt+RYWjrJpa9aLVW0PLgzu3qjItgFF1197jT/yDFesrK66pZpQ7R0aTJtz35HNx3zN7XgEO6Ny1JS+8NS+0/sLLrgzlf29ZBU3SUyPyCzB78fcsXr2V+99ZxWc3nlht/vdFtYHABcJUsLqd7v3UWnvmw1WVtZ6rn1nA6cM6U17hWP79dgYcEjip+1w/i9LyykN+szG226R9i0BNeWbBWi7/52dA4ET9btMuXl38fUSNcOyf5vLg/wxn3KBOocGqsgrH0x9/y/+M7B6x3683xO++WlS0lWN6t6Nbm6xQE3H1lt1c/s/PmDgk9iJVleteXBQRCFp7NaOTBnTg9S8CXUGffL0JIKLJf8+bld1D20pKmf/NZrq1yaJ5k8h/608fjo7ZgRrmVWMDXxV6/uOf8u6XxRHbp/1nMcvWBk72L9ZuC9V4735tOXe/Vtk1F97PG68G36NdM2ZfdRyzFq0lIy2FvVG1sn9+/C1nDO9M5+wsOrbKjHn/H1//kqnj+4UuokHOuZi++K83BE7TYCA98s63gMB5EN6K2LBjD11aZ7FjTxmDbnqNe346lPXbS7hz1jIO65rNr046NJR21N1zOP7QxN8K+Je3V/DkvMovjAqeZyu9C0WwwrF2a+XFPr9wc0SNL79wU2j58283M/ZP78Ycp1PY3+aH93+YMD/hNVuA1lnpbN5VWfbPvo3ttokOAkELvtsad33Qu18Wc97fAl1chdMnst1rcaelRP5f1m8viWjRhAeB/zemN03SUvjD619WeaygFDO6t2vOV97/evGarTHnTbOMNHYmaJl9u2kXTdJSyUxPSdiy3Liz6n758orIc3jAIS1jLt7JCg9iy9dtjxuoLnv6s9By+HlUm5IaIzCzVDNbAKwH3nDOfRwn2RlmVmBmM8ys675maNfeMl5euJoJ977HI3MD/aLhQSCRYP/kNc8uCK37z4LVHPf7Odwxa2lM+l/8I/DHPSxswOy+OStDXRfhKiocMwvWUlHhQh/I4CyB6C6ZeD4t3MSjYX28QMw86PAulw9XBoLjwjh9rW94gSHa2i0lnPHAhxxxx5vM+6qydp+oVtm1TdPQcnQQAHhy3jd84l2gmmWkclTPtnH3U52vN+yk342v8r/PLowJAkFnPDCPkXe9Ffdv+dbSdXEDTHjtceK97/Hr5xaSlREIgJt37Y3Ynjt1ZsQYwYYdgeWfea2Lq59ZwJ2zlgGw8LstnPPYJxFpX/hsdeh1RYXjlLAWX3gQgEC33rUvLGLsn+Yy9k9z+fzbQCv2qLveDqUpKS2PuAi98Hnl/n89oyCmrGP7d6Bl5r49ijgYBE4fekiN3/uPj6r+RsRgEAgKVhbe8lp75RWOZz79lgff+SrmvUG/PKkvV4zpQzB2DDykZcK0AGbQPDOyohPspuqV05whXbLp3LppvLcCgZbmnrJyUlOMPh1aVHms8Ekpgw6pvEB/5VUOW2Sm07dDC8yMIV2ySYmqmAzu3IpeOc2JlpmeSs92zeK2SrdXM6nhlpe/qHL7vkoqEDjnyp1zQ4EuwAgzGxSV5L9ArnNuCPAm8ES8/ZjZJWaWb2YJv6x4wLTXuOaZQO3vjllLuTPORTyevd5FJPxiE961kkjvsH/U2jj9nkO7ZtPzullc/s/P+N1ry0K1j+tfXEzu1JmsifOeYP/23rIK/vHRN/z4wXncPnMpm72LU1l5BUvWRNYgdnn9zyWl5by/MnAhj54xEBzAimfWorWh5Z/9vfLPe32C6ZLBE+7Kf32ecJ9BO/eW82lYrTWac7UzyNz7+tlsLymNCF4r1scf+1lYtIWvN+zk9SXfs2TNNp6bX8RCr7V035xVDI/qOpz6QuXfoaBoC+UVLhToaqLndbN4pWBtwu0vfLY6oi/+h/d/SGFUq3LL7lK27i6lXfOM6LfH1bZZBlt2B86deH3zX981gXNGdic7K36wGNU3h6nj+yd1LICfHRN47HFN+qSdc2z1gk5aqvHzp/Lpdd0sfvv8oohupXD3nz08tBxsxZ4T1SIHuHvykNBysBWY06JJTLrgdTg1iVlbqSlG0/RUhnTJpn/H+MGnZVjASUmxiFYZBCpTTcIu5n07VgaWLq2zMDOaNUmLqeUf2qEFzTPTY1ruUFnBTGRfztlk1GjWkHNuC/AOMC5q/UbnXPAq9QhweIL3P+ycy3PO5cXbHs/DcxPXJqqzMsFFJCh36kye/6yoyjThXTEPvfsVmWmxUfy0wyJrW8Ha3qE3zOaGsMG64LjGRU/mc8YD8yLeE/zQxZst1Dk7UMPJu/3NhPkMr1WG+/en8ecob/Py+PLCNQn3GW5jFReFdduqHsCNdkzvtvx+8hDOOyr2Q//hqo1VBryg/MLNjP7DO1zy1PzQuo+/Tu5DMu0/S+h13axq0+W2zUq4bVi3yKmX5x+dmzDtqD+8E/F6/bYS3lleHGqZVCfb697ZXlLK5Acru4VG983h95OHYGa0zkpni3chPq5Pu4j3t2mWQcdWmZx/dG5oKmXfBLXhwukTuXZCv6TyFW7dtj307hCoVL23YgOvLYnfcg0/7oTBsWNpLTLTWXbbOFbcMT607sd5Xfn8xhN5839PCK3r0DK2GzHYIkxNSSIQhAWLtNTK5fDxgabe/oJjacH9x9sHBFoQQ7pkM6RLdsTAt5nRr2MLOrbKjAgW4RWe3u1jWw71KZlZQzlmlu0tNwXGAsui0oT/R08Dqq3GD+hUdROwNgRr2UE92jXb733O+2pjzLqhXbP5zbi+Sb3/6w07eSesX/2S43sCgUDgnOPlBbEX5mmnDtjH3Eb68vbx5N8wlqbpqTW6cCfyk7wuAIy8662Eg3vRbjxlAE9fNJKf5HXllkmDeP7SoyO2//yp+XFnYD1ybmTd4f53Ek97rC3/umRkwm3to2qkN582kNlXHRex7oRDcyKCyYP/E6gfPZRE5WblHeO5+dQBzLt2DNlZGewtq+Avb68MDSzPuvI4Hr9gBD/JC/TCtg678Bzdqx1N0io/2q29gembTxsYqtW2yIytjT5+wREAMffp9MoJfG6mnTKAZbeNY+FNJ8W8978L1/B5nPGHaH06xL/gBe/zaJGZRmZ6KumpKTx+wRG8fs3xofKFXyyju2HCpXiBoG2zDHLbxv/M7wy7NgRbGU3SUumZU5k+ONYRPFL0DWg1eUx0Rloq7Vtk0iSsItk2rEWYEefeqHitnrqSTIugEzDHzAqATwmMEbxiZrea2Wlemiu9qaULgSuB86vbaWqKcfaR3Zg6vh9n7+Mt/4XTJ1I4fSKvXh35AYyuEU05oivv/GoUb4XVKML9cFhnZl91HNdP6B93Bkt1WmSmcdmo3vwoiZt7RkfVDoMzfSbd9wE9rp0Vd750dBfCkT3a8E7YTKZw4dMSw10+uhcZaSm0a96Elk3T2FYSqGG2yEzjzBHdmHXlcbz3m9ERzXWAbm0ia8VH5LbmoXMOZ+AhLRneLfZY/zdlaKh5v+y2cTHbs6Nu7jm8e2sWTIucCREcWLx0VC/+e8WxLL99HGP7t49brpr458VH1ugmqtZZGWSmx/+IhN+k9OJlgWAWPmPn2vH92LRzL4VhExuibwD818WVgaZw+kTOHBG4qM+/YSxpqSmcf0wPOrVqGuryCW8dt24W+XcMr/RkpqcwKywobQ5rzQVn9nTzAtTRvdry70tG8sWtJzO6b+zf+GfH9AjNDGnZNJ3M9NSIsgcD9JzlVU8Z/bsXZH5xQi8+u/FEPrn+B3HTDelS2Y0yum97Dq2iHz/VAjeMBWvhkfvJpnPrLJqFdb+Ep2ka9X8d0iWbvh1bRMx4CrYsgjX38H31qYUafFZGoNto4CEtI2Y4QqCi0bFlZtLdh/ur2kDgnCtwzg1zzg1xzg1yzt3qrZ/mnHvZW77WOTfQOXeYc260c25Z1XsNuOOHg/nFCb1CNZAbJlb2Y4ZPD1xyy8mh5eviNFv7dWxJ4fSJ/HZcP64e2ydiMA9g+hlDyG3XjJQUY8G0Exk3sGPE9pmL1tK/U0suPr4nZx3ZLdR3l+ydu8F+wj/9dCjnxunu+H1YH2e0kQmer/L1XRN46JzD+XDqmNCMqKDLRveme4Jui3bNA7WIKUdEjtdPHFzZfZWZnsqz+UUMvvl1tpeUkV+4iQGHtKRrmywmDO7EV3dO4DDvA9m6WQY3hbVInvvF0Zw8sCMzrzyOnx4ROyegRWYat50+iMLpE8lMTyX/hrFRc+ZjZz1kZ2WEur+A0FTX84/OZXCXVjRJS0269tXPa3q/95vRPHpuHqvunBDadnSvdrx0+TFVvv/Hh3fhnJHdOaZ3WzLTU/lwavwL1pZdpdx06gD6dWwRuqcj3KEdWsTctxLdZTGsW3aoMgPw23H9+NfFI2nbPLIm2DpO339208gLRPig5I+Gd4noz94S1vo7eWDgnB7Vtz3/N2Uoj5ybx8iebWO6PV687GheuOxopp06gGJvrCo8AATzHfzfBmf/XTP20Ig0X94+nmW3jWNU3/YUTp/IoM6taNMsI+acfuJnI7hsVK+IabXVGdi5VbU9C9F/8545zWmWkRa6HydacOB3SJfs0JhDsCWQYsagzq0Y3LlVqNuoOs2bB/4va9asYfLkyXGP94MxY8jPjxw27diqKWZGZnoq/75kJB9dG/88rC0HxLMaTugbmJ53ePfWPHpuHtdPCASEc4/qTu/2zWnWJI0zhnehZ7tmjB+UeI7+paN6cfXYQ5l8eOJJS9lZGdx75rCIdZ9ePzbi9YJpJ7L89nEcnqB2HW3eqspZOudF9RUP6dKKn+R15Q2viRstUfPPzDh5YEcOyW5K1zZZ3H565fj8CYfmRFwYv7pzAn07tPBulw+s79AyM+IiGH5hiJ5+G13rSkkxXrjsGK4ZeyiPn38EZ47oRosmaTx1YeT9AvEuzi2iZre0a96EGWHdP+MGdYx+CwAfTB0T+r8HhfezVueLW09m9lXH8erVx1M4fSJd22QxdkAHUlOMuycPCQ2AQmSFI9hduOrOCRROn8jdPz6M204fxNMXjQzlYdWdE/jy9vERx9tTVsEFx/Tg1avj/19H92vPz4/vFXodr+kfPWskOyuDo3rFzs7aE2e2VXRLJbzm2CwjlayMtFAL444fVp47P/Y+G4M7t2LS0M4Rtdxww7q1DrX4tntTYpd/HzvNtHXU/6hX+8iumIy0lLizY6KdcGgOvxlX87GJZAw4pCUDvVk/zZuk0at98yq7loIy0wKtn/DZRSlm+/TNYYcccggzZsyoMk2w66tnu8jWxsiebenYKpO5vx6dsCW1vw6IZw2N7tueJbecHHNS3jqp8gT+408OA5K7QzQ1xXjl/x3LKX95n0tH9YrZnpGWwkXH9qB4xx7u+enQmH9sWmoKaVTWrgHe/uUJjPljYH73DRP7c/vMymGQa8MuYNF9r//2+pkTTVWLfg4KVHY1hPufkd0jBp7DpaQYr3mB5iLvkQQdWmaSmmIsu20cRZt3x3xgw43pF9slkJpiofsMABaFtcqqkmia45AurSgo2lrlNMgLj+0RMdU3uq/68tG9KCjaypM/G8HVzyzg4uN6cv2Li3js/CPIykiL6JoJ9+O8yIrB0b0quw7fuOZ49pZXVDnAmJpiEdsnDu7ENSceGjft9B8NDs0pDx9HeOz8QBfKiB5t+OTrTbz/29EJjxctL7ey1ZjTognOxQbh8P9vsJvhrh8N4a4fRbZGxw7owIo7xtfoeV13/HAQ17+4mIuO61lt2pMGdOS+s4bHHYNoKGkp+1bfTUkxukeNMfz2t7+le/fuXHbZZQDcfPPNmBlz585l8+bNlJaWcvvttzNp0qSI9xUWFnLKKaewePFidu/ezQUXXMAXX3xB//792b070ErOykiL6eIK162KyQv764D5byWqmcSki3PXYjyDOrdixi+OStgnfMMp1Q/Aht8Y0zOnOavunECFc+R7dzhD7B2u4Re66G0Lpp3I0Fvf4P+N6c1fvLGA8A/0z4/vGRFUoj12Xl7EgPcj5+bFTBl8c+k6b7+B15npqTEzEpqkpbCnrIIUC9wZeMbhXRIes6YSXQCe+8VRrFi3I2GTHCoH+RL59cmVNcb/mxJo1f3nimNrnMd+HVtw8sAO3DBxQCDoJ3lRnH/DWNLTUqoMZlNGdGOKtzzUm1n0jwuP5Fhv3Orpi46kvMIlVUsO6hj2N4tuvQZFD15XpaYPbTz7yO6cfWRsl2c8GWkpNb6zfp/Nngrf1/LTZDsOhvHTE26eMmUKV199dSgQPPvss7z66qtcc801tGzZkg0bNjBy5EhOO+20hC2HBx54gKysLAoKCigoKGD48OFx09WnAyYQJKsmzbLwmtS+iP6wpqYYqVjC+drh77n4uB4x27KzMkLB4dje7UhPi/xAVveEzB/0jxyzqGoMo6oP+9zfjGbzrr30SzB/el8c16cd763YkLCrq0laKoOSuL0/KN4jJGpLSorx0DlJz2AOie67r0675k1iypGemkINYgCQ3HTI6C65+hR+B3xjN2zYMNavX8+aNWsoLi6mdevWdOrUiWuuuYa5c+eSkpLC6tWrWbduHR07xu8GnTt3LldeeSUAQ4YMYciQxGOI9eWgCwQQuBGlNqaCVmfcoI70ymkW0UUFVBkIINBnX128OjLOnbq3nJb4CzGStey2cbz4+WrOGJ54BlOHlplx52Hvi9euPp7Xl3zPFWN6U1bhfP2I8LpUcPNJcccZorVIsmVdm+6ZMpQB017jsjjdsHWqipp7XZo8eTIzZszg+++/Z8qUKTz99NMUFxczf/580tPTyc3NpaSk6ici78s4Q106KANBvBtR6spbvxwVsy56xka06ro4oo0b2JE3lq4jI23/L6KZ6amcOaL+voGrb8cWoZtk0lP3/+R+9Nw81m1v2MeKH4iSecTEyjvGV5umLmRlpNVpC+5AM2XKFC6++GI2bNjAu+++y7PPPkv79u1JT09nzpw5fPNN1Y/mOP7443n66acZPXo0ixcvpqAg9rEi9e2gDAQNLTM9hbbNMuIORO+LB8+JeyO2L43Vl+3ss2THOmT/DBw4kO3bt9O5c2c6derE2WefzamnnkpeXh5Dhw6lX7+qZz9deumlXHDBBQwZMoShQ4cyYkRyT++tS1ZfX0QSLS8vz0XPnRURqcrSpUvp3z/55yYd7OKV18zm1+QxPclQFUJExOcUCEREfE6BQEQOKg3VnV3f6rOcCgQictDIzMxk48aNjT4YOOfYuHEjmZm1M827Opo1JCIHjS5dulBUVERxcey36jU2mZmZdOlSe3f9V0WBQEQOGunp6fToEXvXvuwfdQ2JiPicAoGIiM8pEIiI+JwCgYiIzykQiIj4XLWBwMwyzewTM1vofUH9LXHSNDGzZ8xspZl9bGa5dZFZERGpfcm0CPYAY5xzhwFDgXFmNjIqzYXAZudcb+DPwO9qN5siIlJXqg0ELmCH9zLd+4m+rW8S8IS3PAP4gR08CaCHAAAOmElEQVRo37wgIiJxJTVGYGapZrYAWA+84Zz7OCpJZ+A7AOdcGbAViP0KLhEROeAkFQicc+XOuaFAF2CEmQ2KShKv9h/zMBAzu8TM8s0s3w+3iIuIHAxqNGvIObcFeAcYF7WpCOgKYGZpQCtgU5z3P+ycy3PO5eXk5OxThkVEpHYlM2sox8yyveWmwFhgWVSyl4HzvOXJwNuusT8eUESkkUjmoXOdgCfMLJVA4HjWOfeKmd0K5DvnXgYeA54ys5UEWgJT6izHIiJSq6oNBM65AmBYnPXTwpZLgB/XbtZERKQ+6M5iERGfUyAQEfE5BQIREZ9TIBAR8TkFAhERn1MgEBHxOQUCERGfUyAQEfE5BQIREZ9TIBAR8TkFAhERn1MgEBHxOQUCERGfUyAQEfE5BQIREZ9TIBAR8TkFAhERn1MgEBHxOQUCERGfUyAQEfG5agOBmXU1szlmttTMlpjZVXHSjDKzrWa2wPuZFm9fIiJy4ElLIk0Z8Evn3Gdm1gKYb2ZvOOe+iEr3nnPulNrPooiI1KVqWwTOubXOuc+85e3AUqBzXWdMRETqR43GCMwsFxgGfBxn81FmttDMZpvZwATvv8TM8s0sv7i4uMaZFRGR2pd0IDCz5sDzwNXOuW1Rmz8DujvnDgP+ArwUbx/OuYedc3nOubycnJx9zbOIiNSipAKBmaUTCAJPO+deiN7unNvmnNvhLc8C0s2sXa3mVERE6kQys4YMeAxY6pz7U4I0Hb10mNkIb78bazOjIiJSN5KZNXQMcA6wyMwWeOuuA7oBOOceBCYDl5pZGbAbmOKcc3WQXxERqWXVBgLn3PuAVZPmr8BfaytTIiJSf3RnsYiIzykQiIj4nAKBiIjPKRCIiPicAoGIiM8pEIiI+JwCgYiIzykQiIj4nAKBiIjPKRCIiPicAoGIiM8pEIiI+JwCgYiIzykQiIj4nAKBiIjPKRCIiPicAoGIiM8pEIiI+JwCgYiIz1UbCMysq5nNMbOlZrbEzK6Kk8bM7F4zW2lmBWY2vG6yKyIita3aL68HyoBfOuc+M7MWwHwze8M590VYmvFAH+/nSOAB77eIiBzgqm0ROOfWOuc+85a3A0uBzlHJJgFPuoCPgGwz61TruRURkVpXozECM8sFhgEfR23qDHwX9rqI2GCBmV1iZvlmll9cXFyznIqISJ1IOhCYWXPgeeBq59y26M1x3uJiVjj3sHMuzzmXl5OTU7OciohInUgqEJhZOoEg8LRz7oU4SYqArmGvuwBr9j97IiJS15KZNWTAY8BS59yfEiR7GTjXmz00EtjqnFtbi/kUEZE6ksysoWOAc4BFZrbAW3cd0A3AOfcgMAuYAKwEdgEX1H5WRUSkLlQbCJxz7xN/DCA8jQMur61MiYhI/dGdxSIiPqdAICLicwoEIiI+p0AgIuJzCgQiIj6nQCAi4nMKBCIiPqdAICLicwoEIiI+p0AgIuJzCgQiIj6nQCAi4nMKBCIiPqdAICLicwoEIiI+p0AgIuJzCgQiIj6nQCAi4nMKBCIiPldtIDCzv5nZejNbnGD7KDPbamYLvJ9ptZ9NERGpK9V+eT3wd+CvwJNVpHnPOXdKreRIRETqVbUtAufcXGBTPeRFREQaQG2NERxlZgvNbLaZDUyUyMwuMbN8M8svLi6upUOLiMj+qI1A8BnQ3Tl3GPAX4KVECZ1zDzvn8pxzeTk5ObVwaBER2V/7HQicc9ucczu85VlAupm12++ciYhIvdjvQGBmHc3MvOUR3j437u9+RUSkflQ7a8jM/gWMAtqZWRFwE5AO4Jx7EJgMXGpmZcBuYIpzztVZjkVEpFZVGwicc2dWs/2vBKaXiojIQUh3FouI+JwCgYiIzykQiIj4nAKBiIjPKRCIiPicAoGIiM8pEIiI+JwCgYiIzykQiIj4nAKBiIjPKRCIiPicAoGIiM8pEIiI+JwCgYiIzykQiIj4nAKBiIjPKRCIiPicAoGIiM8pEIiI+Fy1gcDM/mZm681scYLtZmb3mtlKMysws+G1n00REakrybQI/g6Mq2L7eKCP93MJ8MD+Z0tEROpLtYHAOTcX2FRFkknAky7gIyDbzDrVVgZFRKRu1cYYQWfgu7DXRd46ERE5CNRGILA461zchGaXmFm+meUXFxfXwqFFRGR/1UYgKAK6hr3uAqyJl9A597BzLs85l5eTk1MLhxYRkf1VG4HgZeBcb/bQSGCrc25tLexXRETqQVp1CczsX8AooJ2ZFQE3AekAzrkHgVnABGAlsAu4oK4yKyIita/aQOCcO7Oa7Q64vNZyJCIi9Up3FouI+JwCgYiIzykQiIj4nAKBiIjPKRCIiPicAoGIiM8pEIiI+JwCgYiIzykQiIj4nAKBiIjPKRCIiPicAoGIiM8pEIiI+JwCgYiIzykQiIj4nAKBiIjPKRCIiPicAoGIiM8pEIiI+FxSgcDMxpnZcjNbaWZT42w/38yKzWyB93NR7WdVRETqQrVfXm9mqcB9wIlAEfCpmb3snPsiKukzzrkr6iCPIiJSh5JpEYwAVjrnvnLO7QX+DUyq22yJiEh9SSYQdAa+C3td5K2LdoaZFZjZDDPrWiu5ExGROpdMILA461zU6/8Cuc65IcCbwBNxd2R2iZnlm1l+cXFxzXIqIiJ1IplAUASE1/C7AGvCEzjnNjrn9ngvHwEOj7cj59zDzrk851xeTk7OvuRXRERqWTKB4FOgj5n1MLMMYArwcngCM+sU9vI0YGntZVFEROpStbOGnHNlZnYF8BqQCvzNObfEzG4F8p1zLwNXmtlpQBmwCTi/DvMsIiK1yJyL7u6vH3l5eS4/P79Bji0icrAys/nOubza3KfuLBYR8TkFAhERn1MgEBHxOQUCERGfUyAQEfE5BQIREZ9TIBAR8TkFAhERn1MgEBHxOQUCERGfUyAQEfE5BQIREZ9TIBAR8TkFAhERn1MgEBHxOQUCERGfUyAQEfE5BQIREZ9TIBAR8bmkAoGZjTOz5Wa20symxtnexMye8bZ/bGa5tZ1RERGpG9UGAjNLBe4DxgMDgDPNbEBUsguBzc653sCfgd/VdkZFRKRuJNMiGAGsdM595ZzbC/wbmBSVZhLwhLc8A/iBmVntZVNEROpKMoGgM/Bd2Osib13cNM65MmAr0LY2MigiInUrLYk08Wr2bh/SYGaXAJd4L/eY2eIkjn+wagdsaOhM1JHGXDZQ+Q5mjblsEChf99reaTKBoAjoGva6C7AmQZoiM0sDWgGbonfknHsYeBjAzPKdc3n7kumDQWMuX2MuG6h8B7PGXDYIlS+3tvebTNfQp0AfM+thZhnAFODlqDQvA+d5y5OBt51zMS0CERE58FTbInDOlZnZFcBrQCrwN+fcEjO7Fch3zr0MPAY8ZWYrCbQEptRlpkVEpPYk0zWEc24WMCtq3bSw5RLgxzU89sM1TH+waczla8xlA5XvYNaYywZ1VD5TD46IiL/pERMiIj7XIIGgukdWHCjM7G9mtj58mquZtTGzN8xshfe7tbfezOxer0wFZjY87D3neelXmNl5YesPN7NF3nvurc+b8Mysq5nNMbOlZrbEzK5qZOXLNLNPzGyhV75bvPU9vMegrPAei5LhrU/4mBQzu9Zbv9zMTg5b36DnsZmlmtnnZvZKIyxboXfuLDCzfG9dozg3veNnm9kMM1vmfQaPatDyOefq9YfAgPMqoCeQASwEBtR3PpLM6/HAcGBx2LrfA1O95anA77zlCcBsAvdUjAQ+9ta3Ab7yfrf2llt72z4BjvLeMxsYX49l6wQM95ZbAF8SeIRIYymfAc295XTgYy/fzwJTvPUPApd6y5cBD3rLU4BnvOUB3jnaBOjhnbupB8J5DPwv8E/gFe91YypbIdAual2jODe94z8BXOQtZwDZDVm+eit42B/gKOC1sNfXAtfWdz5qkN9cIgPBcqCTt9wJWO4tPwScGZ0OOBN4KGz9Q966TsCysPUR6RqgnP8BTmyM5QOygM+AIwncbJQWfS4SmBV3lLec5qWz6PMzmK6hz2MC9/O8BYwBXvHy2ijK5h2zkNhA0CjOTaAl8DXeGO2BUL6G6BpK5pEVB7IOzrm1AN7v9t76ROWqan1RnPX1zusqGEag1txoyud1nSwA1gNvEKjlbnGBx6BE5ynRY1JqWu76cg/wG6DCe92WxlM2CDyZ4HUzm2+BJxJA4zk3ewLFwONe196jZtaMBixfQwSCpB5HcRBKVK6arq9XZtYceB642jm3raqkcdYd0OVzzpU754YSqD2PAPpXkaeDpnxmdgqw3jk3P3x1Ffk5aMoW5hjn3HACTz2+3MyOryLtwVa+NAJdzg8454YBOwl0BSVS5+VriECQzCMrDmTrzKwTgPd7vbc+UbmqWt8lzvp6Y2bpBILA0865F7zVjaZ8Qc65LcA7BPpXsy3wGJToPIXKYZGPSalpuevDMcBpZlZI4GnAYwi0EBpD2QBwzq3xfq8HXiQQyBvLuVkEFDnnPvZezyAQGBqufPXZ7+cq+yi/IjA4FRyIGljf+ahBfnOJHCO4m8gBnd97yxOJHND5xFvfhkB/YGvv52ugjbftUy9tcEBnQj2Wy4AngXui1jeW8uUA2d5yU+A94BTgOSIHVC/zli8nckD1WW95IJEDql8RGEw9IM5jYBSVg8WNomxAM6BF2PKHwLjGcm56x38P6Ost3+yVrcHKV68nbdgfYQKBWSqrgOsbIg9J5vNfwFqglECUvZBA3+pbwArvd/APbwS+wGcVsAjIC9vPz4CV3s8FYevzgMXee/5K1OBRHZftWALNxQJggfczoRGVbwjwuVe+xcA0b31PAjMqVhK4cDbx1md6r1d623uG7et6rwzLCZt9cSCcx0QGgkZRNq8cC72fJcHjN5Zz0zv+UCDfOz9fInAhb7Dy6c5iERGf053FIiI+p0AgIuJzCgQiIj6nQCAi4nMKBCIiPqdAICLicwoEIiI+p0AgIuJz/x/KlW8PezPJMgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 2e-4, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('model-60k-vocab')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we train a model with a 30k vocabulary.\n",
    "Because of this, batch size can be higher and training is quicker."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm_full = (TextList.from_csv(PATH, csv_name='fulltrain.csv', cols=0, processor=[TokenizeProcessor(tokenizer=tokenizer), NumericalizeProcessor(max_vocab=30000)])\n",
    "           #Inputs: all the text files in path\n",
    "            .split_from_df(col=1)\n",
    "           #We may have other temp folders that contain text files so we only keep what's in train and test\n",
    "            .label_for_lm()           \n",
    "           #We want to do a language model so we label accordingly\n",
    "            .databunch(bs=64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm_full.save('full_lm_30k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm_full = TextLMDataBunch.load(PATH, 'full_lm_30k', bs=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30002"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_lm_full.train_ds.vocab.itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data_lm_full, drop_mult=0, qrnn=True, callback_fns=ShowGraph)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8HPWd//HXR92SLFe5yb2BweAmTAsmhM4BpoQELiQmARwu5AjJ5XLhchdykFzIkTuSXH53xKEH4iS00EIxXOg2IOMmDMbdlptkJMuSLdkqn98fO8KLkGTZ0u7sat/Px2MeOzvznZ231uWjmfnOd8zdEREROVxpYQcQEZHkpkIiIiJdokIiIiJdokIiIiJdokIiIiJdokIiIiJdokIiIiJdokIiIiJdokIiIiJdkhF2gHgYOHCgjx49OuwYIiJJZfHixTvdvfBg7VKikIwePZqSkpKwY4iIJBUz29iZdjq1JSIiXaJCIiIiXaJCIiIiXaJCIiIiXaJCIiIiXaJCIiIiXaJCIiIiXZIS95EcrseXlLG+Yk/kjVmHbTte231aYhzsCcntxbUOkra/TfvtrJ2NPtEm+ASzyGe1ty76M+3j+ZZt7ON5zEizyLZpFrQJtkkL2rW8pqcZaUH7yKuRnnZgXXrwvmXKSEuLvKYbGWlGZnpaMBlZGZH57Iy0dn9ukVSkQtKBp5Zt46+ryg/6n7aknqyMNHIy0sjJTCc3K51eWRnkZaXTOyeD3jmZFPTKoH9uFgPys+mfl8WwvjmM6JfLwPxs0tJUhKRnUSHpwD1XHdetn+funfpNtr123qqitfdZrdsdWN7BPjvI0l676FUetaat/bhH2nxymwP78Kh2LW9a2ntUm5bPwaE5mG92aG72j7dvdg+myHZN7jQ3f3J5U3NkvqnZaW52Gpsj803BfGNzM41NTkNTMw0fvzazr7FlamJfQzP1DU3s3d8yNbKzdj/rd+6huq6BXXUNn/ousjPSGFeYz5QRfZk2oi/HjenPmIF57Xz7IslBhSSOOns6pP3TRV3dvlObt97qcDYSIsWqau9+Pqrdz9ZddZRV7WVT5V4+2F7D08u3Mv/tTQAcOaQ3F0wZxvnHDmXUABUVST7W3m+vXf5gs3uA84Fyd58cLOsP/BEYDWwAvuDuVa22mwr8L1AANAE/cfc/BuvuA04FqoPmV7n70oNlKS4udo21JYmkudlZt3MPr35YwdPLt/Lupl2YwVUnjeZ7Zx9Jr6z0sCOKYGaL3b34oO1iWEhmAbXAA1GF5D+ASne/zcy+D/Rz939qtd1EwN19tZkNAxYDk9x9V1BInnb3Rw4liwqJJLotu+r4zStreWDhRsYMzOP2zx9L8ej+YceSFNfZQhKz7r/u/ipQ2WrxbOD+YP5+4KI2tvvQ3VcH81uBcuCgwxiLJLOivr24ZfZkfn/t8TQ0NXPZbxby+7c2hR1LpFPifR/JYHffBhC8DuqosZnNBLKAtVGLf2Jmy83sDjPL7mDbuWZWYmYlFRUV3ZFdJOZOGjeQ52+cxakTC/nXJ0p5ffXOsCOJHFTC3pBoZkOB3wFfdffmYPFNwJHAcUB/4J/a2Rx3n+fuxe5eXFioAxpJHnnZGfz3FdMYX5jPNx5azNqK2rAjiXQo3oVkR1AgWgpFeVuNzKwAeAb4F3df1LLc3bd5xD7gXmBmHDKLxF3vnEzumlNMZnoa19xfwq69+8OOJNKueBeSJ4E5wfwc4InWDcwsC3icyEX6h1utaylCRuT6SmlM04qEaET/XO788gzKqvbyz4+vCDuOSLtiVkjMbD6wEDjCzMrM7GrgNuBMM1sNnBm8x8yKzeyuYNMvALOAq8xsaTBNDdY9ZGYrgBXAQODHscovkgiOG92fb3x2PH9ZsZ3SLdUH30AkBDHr/ptI1P1Xktnu+gY+c9v/MXNMf+6a072jLYh0JPTuvyLSPQpyMpk7aywvvl/Oss27wo4j8ikqJCJJ4KqTx9A3N5M7Xvww7Cgin6JCIpIE8rMz+Pqscby8qoLFG6sOvoFIHKmQiCSJr5w4igF5WfxCRyWSYFRIRJJEXnYG15wyltdW7+T9bbvDjiPyMRUSkSRyxcwR5GSmcf+bG8KOIvIxFRKRJNI3N4uLpw3n8SVbqNqju90lMaiQiCSZq04azb7GZua/o9GBJTGokIgkmSOG9OakcQP43cKNNDY1H3wDkRhTIRFJQledNJpt1fW8sHJH2FFEVEhEktHpkwYzon8v7ntjQ9hRRFRIRJJReprxlRNG8/aGSj7Yrq7AEi4VEpEkdcn0IjLSjMeXbAk7iqQ4FRKRJDUgP5tTJxbyxJKtNDf3/FG8JXGpkIgksYumFbF9dz2L1n8UdhRJYSokIknsjEmDyc/O4M86vSUhimkhMbN7zKzczEqjlvU3swVmtjp47dfOtnOCNqvNbE7U8hlmtsLM1pjZr4LH7oqkpF5Z6ZwzeQjPrthOfUNT2HEkRcX6iOQ+4JxWy74PvOTuE4CXgvefYGb9gZuB44GZwM1RBed/gbnAhGBq/fkiKeWSaUXU7Gvkxfd1T4mEI6aFxN1fBSpbLZ4N3B/M3w9c1MamZwML3L3S3auABcA5ZjYUKHD3hR55RvAD7WwvkjKOHzuAIQU5Or0loQnjGslgd98GELwOaqNNEbA56n1ZsKwomG+9/FPMbK6ZlZhZSUVFRbcEF0lE6WnG7KnDeHlVBZUayFFCkKgX29u67uEdLP/0Qvd57l7s7sWFhYXdGk4k0Vw0rYjGZueZFdvCjiIpKIxCsiM4RUXwWt5GmzJgRNT74cDWYPnwNpaLpLRJQwsYPyifZ5brn4PEXxiF5EmgpRfWHOCJNto8D5xlZv2Ci+xnAc8Hp8JqzOyEoLfWV9rZXiTlnHfMUN5eX0l5TX3YUSTFxLr773xgIXCEmZWZ2dXAbcCZZrYaODN4j5kVm9ldAO5eCdwKvBNMtwTLAP4OuAtYA6wFno3lzyCSLM4/dijNDs+Xbg87iqQYi3R+6tmKi4u9pKQk7BgiMXfGf73CwPws/jD3xLCjSA9gZovdvfhg7RL1YruIHIa/OWYob+n0lsSZColID/I3xw7FHZ7T6S2JIxUSkR5k4uDeTBiUzzPL1Q1Y4keFRKSHOe+Yoby9oZLy3Tq9JfGhQiLSw3x8eus9nd6S+FAhEelhWk5vPb1Mp7ckPlRIRHqg848dxjsbK9lerdNbEnsqJCI90PlTIqe3NPaWxIMKiUgPNK4wn6OGFvC0xt6SOFAhEemhzp8ylCWbdrG5cm/YUSQEu+sbWL9zD41NzTHflwqJSA91/jHDAJ3eSlWvrKrgtJ+/zNqKPTHflwqJSA81ckAuU0b01emtFFVWVQdAUb9eMd+XColID3bBsUMp3bKb9Ttj/1upJJYtu/bSNzeT/OyMmO9LhUSkBzvvmKEAPL1MRyWppqyqjuFxOBoBFRKRHm1Y314Uj+rHU8u3kgqPjJADyqrqKOrbgwuJmX3LzErN7D0zu7GN9f9oZkuDqdTMmsysf7Bug5mtCNbpISMiBzF7WhEf7qjlva27w44iceLubKmqY3i/3LjsL+6FxMwmA9cCM4EpwPlmNiG6jbvf7u5T3X0qcBPwStQTEgFOC9Yf9IErIqnugmOHkpWexmPvbgk7isRJ5Z791DU09egjkknAInff6+6NwCvAxR20vwKYH5dkIj1Q39wsTp80iCeWbqEhDvcUSPhaemz15GskpcAsMxtgZrnAecCIthoG688BHo1a7MALZrbYzObGPK1ID3DJ9OF8tGc/r35YEXYUiYMtu1oKSQ89teXu7wM/AxYAzwHLgMZ2ml8AvNHqtNbJ7j4dOBe43sxmtbWhmc01sxIzK6mo0D8eSW2fPaKQ/nlZPPpuWdhRJA7KqiKjGcTjHhII6WK7u9/t7tPdfRZQCaxup+nltDqt5e5bg9dy4HEi11ra2sc8dy929+LCwsLuCy+ShDLT07hwyjBeXFlO9d6GsONIjJVV1dE7J4M+vTLjsr+wem0NCl5HApfQxjUQM+sDnAo8EbUsz8x6t8wDZxE5VSYiB/H5GcPZ39TM0yt0T0lPF88eWxDefSSPmtlK4CngenevMrPrzOy6qDYXAy+4e/QtuYOB181sGfA28Iy7Pxe/2CLJ6+hhBUwcnM+ji3V6q6eL5z0kALG/d74N7n5KG8vubPX+PuC+VsvWEekyLCKHyMy4dPpwfvrsB6wpr2X8oPywI0kMuDtlVXs5cdyAuO1Td7aLpJBLZwwnM9146K2NYUeRGKmua2DP/qa4df0FFRKRlDIwP5tzJg/l0cVl1O1vCjuOxEC87yEBFRKRlHPl8SPZXd/IUxrIsUdq6fqbChfbRSQkM8f0Z+LgfB7U6a0eSUckIhJzZsaXjh/F8rJqlpftCjuOdLOyqjrystLjdg8JqJCIpKSLpxfRKzOdhxZtCjuKdLMtuyL3kJhZ3PapQiKSggpyMpk9dRhPLNtCdZ3udO9Jyqrq4jY0SgsVEpEUdeUJo6hvaOYR3aDYo5RV7Y3r9RFQIRFJWZOL+jBjVD9+t3ADzc16emJPUF3XQE19owqJiMTPnJNGs+Gjvbyi4eV7hC1Bj62ivvHr+gsqJCIp7dzJQxjUO5t739wQdhTpBgfuIdERiYjESWZ6GleeMIpXP6xgbUVt2HGkiw480EqFRETi6IqZI8lKT+MBHZUkvbKqOnIy0+iflxXX/aqQiKS4wt7Z/M2xQ3lkcRk19eoKnMy2V9czrE+vuN5DAiokIkLkovue/U3qCpzktu+uZ3BBTtz3q0IiIkwd0ZepI/py35sbaFJX4KS1vbqeIX1SpJCY2bfMrNTM3jOzG9tY/1kzqzazpcH0w6h155jZKjNbY2bfj29ykZ7rmlPGsPGjvbz0/o6wo8hhaG52ymtS5IjEzCYD1wIziTzt8Hwzm9BG09fcfWow3RJsmw78P+Bc4CjgCjM7Kk7RRXq0c44eQlHfXtz1+vqwo8hhqNy7n4YmZ0hBdtz3HcYRySRgkbvvdfdG4BUiz2fvjJnAGndf5+77gT8As2OUUySlZKSncdVJo3l7fSWlW6rDjiOHaHt1PUDKnNoqBWaZ2QAzywXOA0a00e5EM1tmZs+a2dHBsiJgc1SbsmCZiHSDL84cQV5WOnfrqCTptBSSlDi15e7vAz8DFgDPAcuAxlbN3gVGufsU4L+BPwfL2+rT1uaVQTOba2YlZlZSUaHhH0Q6oyAnky8cN4Knlm39+D8mSQ7bd6fWEQnufre7T3f3WUAlsLrV+t3uXhvM/wXINLOBRI5Aoo9ehgNtPi/U3ee5e7G7FxcWFsbk5xDpib560hia3Hlg4Yawo8gh2LG7njSDwvzUuEaCmQ0KXkcClwDzW60fYsEdNWY2k0jOj4B3gAlmNsbMsoDLgSfjmV2kpxs5IJezjxrCQ29tYu/+1icLJFFtr65nYH42Genx/289rPtIHjWzlcBTwPXuXmVm15nZdcH6zwOlZrYM+BVwuUc0At8EngfeB/7k7u+F8QOI9GTXzhpDdV0DD5foBsVksX13OPeQAGSEsVN3P6WNZXdGzf8a+HU72/4F+Evs0onIjFH9mT6yL3e9vo4rTxhFelp8h9yQQ7djdz2jBuSFsm/d2S4ibZo7ayybK+t4/r3tYUeRTtheXc+QEHpsgQqJiLTjzKOGMGpALr95dR3uGjYlkdXtb2J3fWNop7ZUSESkTelpxjWfGcOyzbso2VgVdhzpQEvX3zDuIQEVEhHpwOdnjKBfbibzXl0XdhTpwMd3tSdyITGzcWaWHcx/1sxuMLO+sY0mImHrlZXOl08czYKVO7j3Dd3tnqh2fHwzYvzvIYHOH5E8CjSZ2XjgbmAM8PuYpRKRhPGNz47jrKMG829PreSXL67W9ZIElCyntpqDezguBn7h7t8GhsYulogkipzMdP7nS9O5dPpw7njxQ255eiXNemZJQtleXU9eVjq9czJD2X9n7yNpMLMrgDnABcGycBKLSNxlpKdx++ePpU+vTO55Yz2Th/Xh0hnDw44lgR276xkcUo8t6PwRyVeBE4GfuPt6MxsDPBi7WCKSaNLSjH89fxJHDunNPHUJTijbd4d3Dwl0spC4+0p3v8Hd55tZP6C3u98W42wikmDMjGtPGcuqHTW8/KFG1U4UO0K8GRE632vrZTMrMLP+RIZ9v9fM/iu20UQkEV0wZRhDCnL4rboEJ4TII3b3JcWprT7uvpvISL33uvsM4IzYxRKRRJWVkcZXTx7Nm2s/0pMUE8DOPftobPbEPyIBMsxsKPAF4OkY5hGRJHDF8SPJz87QjYoJYEf1PiC8rr/Q+UJyC5Gh29e6+ztmNpZWD6MSkdRRkJPJFTNH8MyKbZRV7Q07TkoL88mILTp7sf1hdz/W3f8ueL/O3S+NbTQRSWRf+8wYDLjrNd3xHqaPC0miH5GY2XAze9zMys1sh5k9ambqRC6Swob26cWFU4fxx3c2U7Vnf9hxUtaO6sgjdgfmZ4WWobOntu4l8kjbYUARkScb3nu4OzWzb5lZqZm9Z2Y3trH+S2a2PJjeNLMpUes2mNkKM1tqZiWHm0FEuu7rs8ZR19DEAws3hh0lZW3fXU9h73Aesduis3sudPd73b0xmO4DCg9nh2Y2GbgWmAlMAc43swmtmq0HTnX3Y4FbgXmt1p/m7lPdvfhwMohI9zhiSG8+d+Qg7l+4gbr9TWHHSUk7dtczpE+vUDN0tpDsNLMrzSw9mK4EPjrMfU4CFrn73mD8rleIjOH1MXd/091bHoCwCNBpNJEEdd2p46jcs5+HF28OO0pK2rKrjqEhXh+BzheSrxHp+rsd2AZ8nsiwKYejFJhlZgPMLBc4DxjRQfurgWej3jvwgpktNrO5h5lBRLrJcaP7MX1kX+a9uo7Gpuaw46SUhqZmNn20l7GF4TyrvUVne21tcvcL3b3Q3Qe5+0VEbk48ZO7+PvAzYAHwHJE75RvbamtmpxEpJP8Utfhkd58OnAtcb2az2tl2rpmVmFlJRYWGchCJFTPjulPHUVZVxzMrtoUdJ6Vs/GgPjc3O+EH5oeboytWZ7xzuhu5+t7tPd/dZQCVt3JNiZscCdwGz3f2jqG23Bq/lwONErrW0tY957l7s7sWFhYd1OUdEOumMSYMZV5jHna9oMMd4WlNeC5DUhcQOe0OzQcHrSCJHNvNbrR8JPAZ82d0/jFqeZ2a9W+aBs4icKhOREKWlGV8/dRzvb9vNq6t3hh0nZbQUknGFyVtIuvJrx6NmtpJIN+Lr3b3KzK4zs+uC9T8EBgD/06qb72DgdTNbBrwNPOPuz3Uhh4h0k4umFjGkIIc7X14bdpSUsbq8lqK+vcjL7uyjpWKjw72bWQ1tFwwDDru/mbuf0sayO6PmrwGuaaPNOiJdhkUkwWRlpHHNKWP48TPvs3TzLqaO6Bt2pB5vTXkt40I+rQUHOSJx997uXtDG1Nvdwy2BIpJwLp85koKcDB2VxEFzs7O2opbxIZ/Wgq6d2hIR+YT87AzmnDSa51duZ21FbdhxerQtu+qob2gO/UI7qJCISDebc9JostLTmPeKhpiPpTUVidFjC1RIRKSbDczP5ovHjeCxJWVsq64LO06PtTZBuv6CComIxMC1p4zFHX6jo5KYWVNeS/+8LPrnhTfqbwsVEhHpdiP653LxtCLmv72Jipp9YcfpkVaXJ8aFdlAhEZEY+cZp42loauau13VU0t3cnTXltYwfrEIiIj3YmIF5XDBlGA8u3KgHX3WznbX7qa5r0BGJiPR81582nj37m7j3zQ1hR+lREmWMrRYqJCISMxMH9+aco4dw7xvr2V3fEHacHiORuv6CComIxNg3PzeemvpG7ntjQ9hReoy15bXkZaUztE+4D7RqoUIiIjE1uagPZx41mN++to7qOh2VdIeWMbbMDnsQ9m6lQiIiMXfjGROoqW/k7tfXhx2lR1hdXpMwF9pBhURE4uDoYX04d/IQ7nl9Pbv2qgdXV+yub2DH7n0JMepvCxUSEYmLG8+YyJ79jfz2Nd1X0hWrd9QAcOSQ3iEnOUCFRETi4oghvfmbY4Zy7xsbqNR9JYftg+2RQjJxcIoXEjP7lpmVmtl7ZnZjG+vNzH5lZmvMbLmZTY9aN8fMVgfTnPgmF5GuuPGMCdQ1NHHnK3peyeFatb2GvKx0hvc77GcLdru4FxIzmwxcC8wk8rTD881sQqtm5wITgmku8L/Btv2Bm4Hjg+1vNrN+cYouIl00flBvLppaxP1vbqB8d33YcZLSqu01TBzSO2F6bEE4RySTgEXuvtfdG4FXgItbtZkNPOARi4C+ZjYUOBtY4O6V7l4FLADOiWd4EemaG8+YQFOz8+u/rgk7StJxd1btqEmo6yMQTiEpBWaZ2QAzywXOA0a0alMEbI56XxYsa2+5iCSJUQPyuKx4BPPf3sTmyr1hx0kq5TX72LW3gSMS6PoIhFBI3P194GdEjiaeA5YBja2atXXM5h0s/xQzm2tmJWZWUlFR0YXEItLdbjh9PGbGr15aHXaUpLKq5UK7jkjA3e929+nuPguoBFr/bSrjk0cpw4GtHSxvax/z3L3Y3YsLCwu7L7yIdNnQPr248vhRPPpumZ7tfghaCsmRQwpCTvJJYfXaGhS8jgQuAea3avIk8JWg99YJQLW7bwOeB84ys37BRfazgmUikmS+cdo4cjLTuWPBh2FHSRqrdtRQ2Ds7IZ6KGC2s+0geNbOVwFPA9e5eZWbXmdl1wfq/AOuANcBvgW8AuHslcCvwTjDdEiwTkSQzMD+br508hqeXb6N0S3XYcZLCqu01CXd9BMI7tXWKux/l7lPc/aVg2Z3ufmcw7+5+vbuPc/dj3L0katt73H18MN0bRn4R6R5zTx1L39xMfvbcB2FHSXhNzc6HO2o4IsGuj4DubBeREBXkZHL9Z8fz2uqdvLlmZ9hxEtqmyr3sa2xWIRERae3LJ45iWJ8cfvbcB7i32QlTgFXbdwPo1JaISGs5menceOZElpVV81zp9rDjJKwPttdgllhjbLVQIRGR0F06fTgTBuVz+/OraGxqDjtOQvpwRw2j+ufSKys97CifokIiIqFLTzO+d86RrNu5hz+WbD74Binog+01CXk0AiokIpIgzpg0iJmj+3PHgtXs2dd6sIvUVt/QxIadexJujK0WKiQikhDMjJvOO5Kdtfv08KtW1pTX0uyJNzRKCxUSEUkY00b247xjhjDv1XWU12iY+RaryyNDoyRijy1QIRGRBPOPZx/J/sZmDegYZfWOWjLSjNED88KO0iYVEhFJKGMG5vGl40cy/+3NrAl+E091q8trGT0wj8z0xPwvOzFTiUhKu+H0CeRlpfOjJ1fqJkVgbXktEwblhx2jXSokIpJwBuRn892zj+D1NTtT/ibFfY1NbPhoD+NVSEREDs3fzhzJpKEF3Pr0SvbuT93uwOt37qHZUSERETlUGelp3Dr7aLZW1/M/f10bdpzQrCmPPPhrwqDE7LEFKiQiksCKR/fnkmlFzHt1Het37gk7TihW76jFDMYWJmaPLVAhEZEE9/1zjyQrI41/e+q9lLzwvqa8lpH9c8nJTLwxtlqE9ajdb5vZe2ZWambzzSyn1fo7zGxpMH1oZrui1jVFrXsy/ulFJJ4GFeTw7TMn8vKqCp5/b0fYceJuTYL32IIQComZFQE3AMXuPhlIBy6PbuPu33b3qe4+Ffhv4LGo1XUt69z9wrgFF5HQzDlxFEcO6Z1yF94bm5pZt7OW8Ql8fQTCO7WVAfQyswwgF9jaQdsrgPlxSSUiCSkjPY1bZk9my646fv1/a8KOEzcbK/fS0OQJ3WMLQigk7r4F+DmwCdgGVLv7C221NbNRwBjg/6IW55hZiZktMrOLYh5YRBLCzDH9uWR6Eb99bR1rK2rDjhMXB3psqZB8gpn1A2YTKRDDgDwzu7Kd5pcDj7h7U9Syke5eDPwt8AszG9fOfuYGBaekoqKiG38CEQnLTedOIicznR8+UZoSF95bCsk4FZJPOQNY7+4V7t5A5PrHSe20vZxWp7XcfWvwug54GZjW1obuPs/di929uLCwsLuyi0iICntn892zjuCNNR/x9PJtYceJudU7ahjWJ4f87Iywo3QojEKyCTjBzHLNzIDTgfdbNzKzI4B+wMKoZf3MLDuYHwicDKyMS2oRSQhXnjCKo4cV8ONnVlLbwx+AtaailvEJOnR8tDCukbwFPAK8C6wIMswzs1vMLLoX1hXAH/yTx6+TgBIzWwb8FbjN3VVIRFJIeprx44smU16zj18s+DDsODHT3OxJ0fUXIr2n4s7dbwZubrX4h63a/KiN7d4EjoldMhFJBtNG9uPy40Zy75sb+HzxcI4cUhB2pG63ZVcd9Q3NCd9jC3Rnu4gkqe+dfQQFORn8y+OlNDf3vAvvydJjC1RIRCRJ9cvL4qZzJ1GysYqHF28OO063a3m8ro5IRERi6LLi4cwc3Z+fPvsBH9XuCztOt3pv626GFOTQNzcr7CgHpUIiIknLzPjJxZPZs6+Rn/zlU50/k9qSTbuYNrJv2DE6RYVERJLahMG9mTtrLI+9u4U31+4MO0632Fm7j02Ve1VIRETi5e8/N4GR/XP5l8dLqW9oOvgGCW7ppsiA59NG9gs5SeeokIhI0svJTOfHF01m3c49/Pz5VWHH6bIlm6vISDMmD+sTdpROUSERkR5h1sRCvnzCKO56fT1vrEnuU1xLN+9i0tACemUl7sOsoqmQiEiP8c/nTWJsYR7/8KdlVO9tCDvOYWlqdpZtrk6a6yOgQiIiPUivrHR++cVp7Kzdxw/+vCIpRwheU15L7b5Gpo5QIRERCcUxw/vw7TMn8vTybTyxtKNn5iWmJZuqgOS50A4qJCLSA1136jiKR/XjX58oZcuuurDjHJIlm3bRNzeT0QNyw47SaSokItLjpKcZd3xxKs3Nzj/8aWlSjcW1ZHMV00b0JfKUjeSgQiIiPdKI/rncfOHRLFpXyd2vrw87TqfU1Dewurw2qU5rgQqJiPRgl80YzllHDeb251fx/rbdYcc5qOVl1biTVD22QIVERHowM+OnlxxDQa9Mbpi/hL37E/uJiks2VWEGU5KoxxZAAiwdAAAO1UlEQVSEVEjM7Ntm9p6ZlZrZfDPLabX+KjOrMLOlwXRN1Lo5ZrY6mObEP72IJJMB+dn84otTWVNRy4+efC/sOB16d9MuxhXmU5CTGXaUQxL3QmJmRcANQLG7TwbSgcvbaPpHd58aTHcF2/Yn8mTF44GZwM1mllwnE0Uk7j4zYSDfPG08fyop4/ElZWHHaVNNfQOvr9nJZ8YPDDvKIQvr1FYG0MvMMoBcoLOdvc8GFrh7pbtXAQuAc2KUUUR6kG+dPoGZo/vzg8dLWVtRG3acT1mwcgf7G5u5YMqwsKMcsrgXEnffAvwc2ARsA6rd/YU2ml5qZsvN7BEzGxEsKwKiH4VWFiz7FDOba2YlZlZSUVHRjT+BiCSjjPQ0fnnFVLIz0rj+oXfZsy+xrpc8uWwrRX17MT3JLrRDOKe2+gGzgTHAMCDPzK5s1ewpYLS7Hwu8CNzfsnkbH9lmB3F3n+fuxe5eXFhY2D3hRSSpDe3Ti19ePo0Pd9TwnQS6v6Ryz35eX72TC6YMS6r7R1qEcWrrDGC9u1e4ewPwGHBSdAN3/8jdW56b+VtgRjBfBoyIajqczp8WExFh1sRC/vm8STz/3g5+8dLqsOMA8GzpNhqbnQumDA07ymEJo5BsAk4ws1yLlN7TgU88I9PMor/NC6PWPw+cZWb9giObs4JlIiKddvVnxnDZjOH86qXVPLN8W9hxeGrZVsYW5nHU0IKwoxyWjHjv0N3fMrNHgHeBRmAJMM/MbgFK3P1J4AYzuzBYXwlcFWxbaWa3Au8EH3eLu1fG+2cQkeRmZvz44siDsP7h4aUMLsimeHT/ULLs2F3PW+sr+dbpE5LytBaAJeMwy4equLjYS0pKwo4hIglmZ+0+vnDnQipq9vH7a0/gmOHxfyLh3a+v59anV/LSP5zKuML8uO+/I2a22N2LD9ZOd7aLSMoamJ/Ng9ccT0GvTL5yz1t8uKMm7hmeWraVo4cVJFwRORQqJCKS0ob17cXvrz2ezPQ0rrzrLdbF8R6T97ftZunmXcyemnz3jkRTIRGRlDdqQB4PXXM8Tc3OF36zkJVb4zPA471vrKdXZjpfKB5x8MYJTIVERASYMLg3f/z6iWSmp3H5vIUs3hjbfjw7a/fx56VbuXRGEX1zs2K6r1hTIRERCYwflM/D151I/7wsrrzrbV5cuSNm+3pw0Ub2Nzbz1ZPHxGwf8aJCIiISZXi/XP503YmMGZjHNQ+UcNNjK7p9OJV9jU08uGgjpx1RmNQX2VuokIiItDKodw6PX38SX581lj+8s4nzfvUaJRu671TXk0u3srN2P1d/Zmy3fWaYVEhERNqQnZHOTedN4g/XnkBTs3PZbxbyr38upaa+oUuf6+7c/fp6jhjcm5PHD+imtOFSIRER6cDxYwfw3I2zmHPiaB58ayNn/terPFe6jcO5mbt6bwP/9OhyPthew9c+Mzpp72RvTXe2i4h00pJNVdz02Ao+2F7DxMH5XP2ZMcyeWkROZnqH27k7z5Zu54dPvEfV3v1cc8oYvnf2kaSnJXYh6eyd7SokIiKHoKGpmSeXbuXu19ezcttuBuRlcc7kIZx19BBOHDuArIwDJ3p27K7nyaVbeXzJFlZu283kogJuu+RYJhfFfyiWw6FCEkWFRES6m7uzcN1HPLhoIy+vqmDv/iZ6Z2dQWJBNc7PT5M6WqjqaHaaM6MtlM4Zz+XEjyEhPnisKnS0kcR/9V0SkJzAzTho3kJPGDaS+oYk31uzkpQ/Kqa5rIN2M9DTjkmm5zJ46jLE9oItvR1RIRES6KCczndMnDeb0SYPDjhKK5DnGEhGRhKRCIiIiXRJKITGzb5vZe2ZWambzzSyn1frvmNlKM1tuZi+Z2aiodU1mtjSYnox/ehERiRb3QmJmRcANQLG7TwbSgctbNVsSrD8WeAT4j6h1de4+NZgujEtoERFpV1intjKAXmaWAeQCW6NXuvtf3X1v8HYRMDzO+UREpJPiXkjcfQvwc2ATsA2odvcXOtjkauDZqPc5ZlZiZovM7KL2NjKzuUG7koqKim7JLiIinxbGqa1+wGxgDDAMyDOzK9tpeyVQDNwetXhkcIPM3wK/MLNxbW3r7vPcvdjdiwsLC7v1ZxARkQPCOLV1BrDe3SvcvQF4DDipdSMzOwP4AXChu+9rWe7uW4PXdcDLwLR4hBYRkbaFcUPiJuAEM8sF6oDTgU+MX2Jm04DfAOe4e3nU8n7AXnffZ2YDgZP55IX4Ni1evHinmW0M3vYBqjuYb/06ENh5CD9f9Gd2Zl3rZe297yhrmBkT/TvsTNbMQ8wXj4yJ/h22XqbvsGd+hxM6lcLd4z4B/wZ8AJQCvwOygVuIHH0AvAjsAJYG05PB8pOAFcCy4PXqw9j3vI7m23gtOdzP78y61svae3+QrKFlTPTvsDNZDzWfvsNPL9N32LO/w4NNoQyR4u43Aze3WvzDqPVntLPdm8AxXdz9UweZb/3alc/vzLrWy9p731HWQ9WdGRP9O+xM1umdj9Zhjo7W9bTvsPUyfYc9+zvsUEqM/tsVZlbinRj9MkyJnlH5ui7RMyZ6Pkj8jImeryMaIuXg5oUdoBMSPaPydV2iZ0z0fJD4GRM9X7t0RCIiIl2iIxIREemSlCokZnaPmZWbWelhbDvDzFaY2Roz+5WZWbD8j1GDSG4ws6WJlC9Y9/dmtioYKPOg3aXjndHMfmRmW6K+x/MSKV/U+u+amQddzw9bjL7DW4NBTpea2QtmNizB8t1uZh8EGR83s76Hmy+GGS8L/o00m9lhXavoSq52Pm+Oma0OpjkH+xlCc6jdzZJ5AmYR6RlRehjbvg2cCBiRIVvObaPNfwI/TKR8wGlEulNnB+8HJdp3CPwI+G4i/xkDI4DngY3AwETLCBREtbkBuDPB8p0FZATzPwN+loDf4STgCCI3OhfHM1ewz9GtlvUH1gWv/YL5fgf7uxrGlFJHJO7+KlAZvczMxpnZc2a22MxeM7MjW29nZkOJ/ENd6JE/xQeAi1q1MeALwPwEy/d3wG0ejA7gUTd4JlDGbhPDfHcA3wO6fFExFhndfXdU07yu5IxRvhfcvTFo2uWBWGOU8X13XxVGrnacDSxw90p3rwIWAOfE69/SoUipQtKOecDfu/sM4LvA/7TRpggoi3pfFiyLdgqww91XJ1i+icApZvaWmb1iZsd1c77uyAjwzeC0xz0WGcEgYfKZ2YXAFndf1s25ui1jkPMnZrYZ+BJR92UlSr4oX+OTA7F2l+7MGO9cbSkCNke9b8kaxs/QoZR+ZruZ5RO5W/7hqFOM2W01bWNZ69/4rqALRyNt6aZ8GUQOi08AjgP+ZGZjg99kEiXj/wK3Bu9vJXKK8GuJkM8iQ/n8gMipmZjorr+H7v4D4AdmdhPwTT5902+o+YLP+gHQCDzUHdlikTFeuczsq8C3gmXjgb+Y2X4iYxFe3EHWuP4MnZHShYTIEdkud58avdDM0oHFwdsnifxHF30oPpyoZ6hY5LkqlwAzEjBfGfBYUDjeNrNmImMOddfY+l3O6O47orb7LfB0N2XrjnzjiIxUvSz4j2A48K6ZzXT37QmSsbXfA8/QTYWku/IFF4vPB07vrl9kujtjDLSZC8Dd7wXuDXK+DFzl7huimpQBn416P5zItZQy4vszHFyYF2jCmIDRRF0IA94ELgvmDZjSznbvEPmtvuXi1nlR684BXknEfMB1wC3B/EQih8qWYBmHRrX5NvCHRMrXqs0GunixPUbf4YSoNn8PPJJg+c4BVgKFXf3uYv3nTBcuth9uLtq/2L6eyBmFfsF8/87+XY3nFNqOQ/lhI6eetgENRKr61UR+23yOyECQK2mn1xWR56KUAmuBXxP1nzFwH3BdIuYDsoAHg3XvAp9LwIy/IzII53IivzUOTaR8rdpsoOu9tmLxHT4aLF9OZHykogTLt4bILzEtA7Eedq+yGGa8OPisfUQGjX0+Xrloo5AEy78WfHdrgK8eyt/VeE66s11ERLpEvbZERKRLVEhERKRLVEhERKRLVEhERKRLVEhERKRLVEgkJZlZbZz3d5eZHdVNn9VkkVF+S83sKTvISLpm1tfMvtEd+xZpi7r/Skoys1p3z+/Gz8vwA4MSxlR0djO7H/jQ3X/SQfvRwNPuPjke+ST16IhEJGBmhWb2qJm9E0wnB8tnmtmbZrYkeD0iWH6VmT1sZk8BL5jZZ83sZTN7xCLP3ngoGBWaYHlxMF8bDLC4zMwWmdngYPm44P07ZnZLJ4+aFnJgcMl8M3vJzN61yLMqZgdtbgPGBUcxtwdt/zHYz3Iz+7du/BolBamQiBzwS+AOdz8OuBS4K1j+ATDL3acRGVX336O2ORGY4+6fC95PA24EjgLGAie3sZ88YJG7TwFeBa6N2v8vg/0fdOykYByp04mMBgBQD1zs7tOJPIfmP4NC9n1grbtPdfd/NLOzgAnATGAqMMPMZh1sfyLtSfVBG0WinQEcFTVKa4GZ9Qb6APeb2QQio6xmRm2zwN2jnz/xtruXAVjkaZmjgddb7Wc/BwamXAycGcyfyIHnSvwe+Hk7OXtFffZiIs+pgMi4S/8eFIVmIkcqg9vY/qxgWhK8zydSWF5tZ38iHVIhETkgDTjR3euiF5rZfwN/dfeLg+sNL0et3tPqM/ZFzTfR9r+xBj9wcbK9Nh2pc/epZtaHSEG6HvgVkeeQFAIz3L3BzDYAOW1sb8BP3f03h7hfkTbp1JbIAS8QeY4HAGbWMvR3H2BLMH9VDPe/iMgpNYDLD9bY3auJPFb3u2aWSSRneVBETgNGBU1rgN5Rmz4PfC14VgZmVmRmg7rpZ5AUpEIiqSrXzMqipu8Q+U+5OLgAvZLIEPwA/wH81MzeANJjmOlG4Dtm9jYwFKg+2AbuvoTIqLKXE3lYVLGZlRA5OvkgaPMR8EbQXfh2d3+ByKmzhWa2AniETxYakUOi7r8iCSJ4GmOdu7uZXQ5c4e6zD7adSNh0jUQkccwAfh30tNpFNz1uWCTWdEQiIiJdomskIiLSJSokIiLSJSokIiLSJSokIiLSJSokIiLSJSokIiLSJf8fsKMq41FHprcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot(skip_start=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 5:53:45\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      3.479924    3.606225    0.315127  (50:35)\n",
      "2      3.308489    3.457322    0.325741  (50:32)\n",
      "3      3.250602    3.386342    0.332229  (50:29)\n",
      "4      3.290317    3.325020    0.339061  (50:29)\n",
      "5      3.190439    3.263015    0.346897  (50:31)\n",
      "6      3.178667    3.212496    0.354239  (50:31)\n",
      "7      3.150851    3.193054    0.357078  (50:33)\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VPW9//HXdyaTPUAIAQIICYqAKEZEiiKIO4tbr7TFVmult1Rt69JN/OnPrWrVenurD21dWqptqdhirTuuWNwQg7JEFtkhQkgAIfs63/vHTIaZkEkmIWeSA+/n45FHzpz5zjmfnMy858z3fM8ZY61FRETcw9PVBYiISPsouEVEXEbBLSLiMgpuERGXUXCLiLiMgltExGUU3CIiLqPgFhFxGQW3iIjLJDix0J6Zve2wo4c6sWgRkcPSsmXLdltrs2Np60hw9x1wFAUFBU4sWkTksGSM2RprW0e6SnT5ExER5zjUx63kFhFxijN73E4sVEREAIf6uJXcItIe9fX1FBUVUVNT09WlOC45OZlBgwbh8/k6vAxHglu5LSLtUVRUREZGBrm5uRhjurocx1hr2bNnD0VFReTl5XV4ORrHLSJdrqamhqysrMM6tAGMMWRlZR3yJwv1cYtIt3C4h3aTzvg7HQnu+ka/E4sVEREcCu7d5bVOLFZExBH79u3j97//fbsfN23aNPbt2+dARa1zJLiz0pOcWKyIiCOiBXdjY2Orj3v11Vfp1auXU2VF5cyoEp06KSIuMmfOHDZu3Eh+fj4+n4/09HRycnJYvnw5q1ev5pJLLmH79u3U1NRw/fXXM3v2bAByc3MpKCigoqKCqVOncvrpp/Phhx8ycOBAXnjhBVJSUhyp15HgblRwi0gH3fnS56zeUdapyzxuQA9uv3BU1Pvvu+8+CgsLWb58Oe+++y7Tp0+nsLAwNGRv7ty59O7dm+rqak455RQuvfRSsrKyIpaxfv16nnnmGZ588km++c1v8txzz3H55Zd36t/RxJHg3ldV78RiRUTiYty4cRHjrB9++GGef/55ALZv38769esPCu68vDzy8/MBOPnkk9myZYtj9Tlz5qSISAe1tmccL2lpaaHpd999l7feeouPPvqI1NRUJk+e3OI47KSkA8f2vF4v1dXVjtXnyMHJFJ/XicWKiDgiIyOD8vLyFu/bv38/mZmZpKamsnbtWpYsWRLn6g7m0MFJJ5YqIuKMrKwsJkyYwPHHH09KSgr9+vUL3TdlyhQee+wxRo8ezfDhwxk/fnwXVhpgnBgBkjFouC0vWtfpyxWRw9OaNWsYOXJkV5cRNy39vcaYZdbasbE83pGuEr92uUVEHBNTcBtjehljFhhj1hpj1hhjTm2tfYNfwS0i4pRY+7gfAhZaa2cYYxKB1LYeYK09Yi4aIyIST23ucRtjegCTgD8BWGvrrLVtnpxfVdf6qaIiItIxsXSVDAVKgT8bYz4zxvzRGJPW1oMU3CIizogluBOAMcAfrLUnAZXAnOaNjDGzjTEFxpgCgGoFt4iII2IJ7iKgyFr7cfD2AgJBHsFa+4S1dmzTcJb91TrtXUQOX+np6QDs2LGDGTNmtNhm8uTJFBQUdPq62wxua20xsN0YMzw462xgdVuPe+7TokMsTUSk+xswYAALFiyI6zpjHVXyE2BecETJJuCqth4won/GodQlIhJXN910E0OGDOHaa68F4I477sAYw+LFi/nqq6+or6/n7rvv5uKLL4543JYtW7jgggsoLCykurqaq666itWrVzNy5EjHrlcSU3Bba5cDMZ3R02TOv1Yxc9zgDhUlIkew1+ZA8arOXWb/E2Dqfa02mTlzJjfccEMouP/xj3+wcOFCbrzxRnr06MHu3bsZP348F110UdShzn/4wx9ITU1l5cqVrFy5kjFjDupV7hS6OqCICHDSSSdRUlLCjh07KC0tJTMzk5ycHG688UYWL16Mx+Phyy+/ZNeuXfTv37/FZSxevJjrrrsOgNGjRzN69GhHanUkuJN9Xs4Z2a/thiIizbWxZ+ykGTNmsGDBAoqLi5k5cybz5s2jtLSUZcuW4fP5yM3NbfGSruHiceKhI9cq8RpDRa1GlYiIu8ycOZP58+ezYMECZsyYwf79++nbty8+n49FixaxdevWVh8/adIk5s2bB0BhYSErV650pE5H9rg9HqiobXBi0SIijhk1ahTl5eUMHDiQnJwcvvOd73DhhRcyduxY8vPzGTFiRKuPv+aaa7jqqqsYPXo0+fn5jBs3zpE6HQlurzGU1yi4RcR9Vq06cGC0T58+fPTRRy22q6ioAAJfGFxYWAhASkoK8+fPd7xGZ7pKPIa9lXVOLFpE5IjnSHBX1jZSXtPAjn3OfeeaiMiRypHgrmkIXKfk/oVrnVi8iByGnPg2ru6oM/5OR4I7Nytw8cDTjs5qo6WICCQnJ7Nnz57DPryttezZs4fk5ORDWo4jBydTEr3UEugyERFpy6BBgygqKqK0tLSrS3FccnIygwYNOqRlOBLcCR5DUoKH4rLWB6qLiAD4fD7y8vK6ugzXcKSrBCCnZ7IOToqIOMCx4E5LSuC99budWryIyBHLsYtMfb6jDID6Rj8+r2PvDyIiRxzHE/Wavy1zehUiIkcUx4L77z/4GgD+w3t0j4hI3DkW3CcPyYz4LSIincOx4E5K8JKU4KFMXxosItKpHO3jrm3w8/jiTU6uQkTkiBOX4R51Df54rEZE5IgQl+C+dt6n8ViNiMgRwdHg/vGZxwDw1ppdTq5GROSI4mhw//z84U4uXkTkiBTTmZPGmC1AOdAINFhrxzpZlIiIRNeeU97PtNbq4iMiIl3M8YOTV4wfAsCK7fucXpWIyBEh1uC2wBvGmGXGmNntWcEnW/YCcPGjH7SzNBERaUmsXSUTrLU7jDF9gTeNMWuttYvDGwQDfTbA4MGDQ/PXFpd3Vq0iIkKMe9zW2h3B3yXA88C4Fto8Ya0da60dm52dHZq/5OazO6lUERGBGILbGJNmjMlomgbOAwpjXUH/nge+FLO8RtctERE5VLHscfcD3jfGrACWAq9Yaxd2ZGV3v7ymIw8TEZEwbQa3tXaTtfbE4M8oa+097V3JnReNAuDZgu0dKFFERMLF5Vol4/J6h6at1TcriIgcirgE98icHqHpF1fsiMcqRUQOW3H7Ft8R/TMAuH7+8nitUkTksBS34H7pJ6eHpl9btTNeqxUROezELbh93gOrukbX5xYR6bC4BTfAOSP7xXN1IiKHpbgG9x+vPHA12L2VdfFctYjIYSOuwR1uzK/e7KpVi4i4WpcFN0DRV1VduXoREVeKe3C/9OMDo0tOv39RvFcvIuJ6cQ/uEwb1jLi9ZmdZvEsQEXG1LukqOaZvemh66kPvUdfg74oyRERcqUuC+7XrJ0bcvvyPH3dFGSIirtQlwe3zeth477TQ7aXBrzcTEZG2ddmoEq/HRIT3T575rKtKERFxlS4dDuj1mND0Syt2hL5YWEREouvS4G7uF/9coet1i4i0ocuD+y+zDnzv8JY9VeTd/GoXViMi0v11eXBPOjab5645ravLEBFxjS4PboCTh2RG3P7u3KVdVImISPfXLYIb4IkrTg5NL/6iVH3dIiJRdJvgPm9Uf3ql+kK3T7zzDYW3iEgLuk1wA9w6/bjQdFlNAxMf0EWoRESa61bBPePkQZw9om/odtFX1V1YjYhI9xRzcBtjvMaYz4wxLztZ0M3TRkTcVneJiEik9uxxXw+scaqQJsf0zeCdn50Run3Zk0ucXqWIiKvEFNzGmEHAdOCPzpYTMDT7wGVfl2zaS+6cVyivqY/HqkVEur1Y97h/B/wSiHrhbGPMbGNMgTGmoLS09JALm3RsdsTtE+54g+q6xkNeroiI27UZ3MaYC4ASa+2y1tpZa5+w1o611o7Nzs5urWlMnvreKQfNG3nbQmobFN4icmSLZY97AnCRMWYLMB84yxjzN0erAjxhVw4MN/zWhWzZXcmyrV85XYKISLfUZnBba2+21g6y1uYCM4F3rLWXO15ZmOanxE9+8F0u/cOHGnEiIkekbjWOu7lN907jt988keeuOY3vn5530P1ri8u7oCoRka5lnNhrHTt2rC0oKOj05X6wYTffaeH7KT+/83zSkhI6fX0iIvFijFlmrR0bS9tuvcfd3IRj+rQ4//zfLeZ3b32B36+uExE5/Llqj7uJtbbVL1xYeMNERvTv4dj6RUQ622G7x93EGMMpuZlR75/yu/cAqKxt4NlPtukgpogcVlzbMfzwZSfx5w+28MTiTS3enzvnldD0gF4pTBx26GPLRUS6A1d2lYTbVFrBWf/zn5jarrjtPHqGXfNbRKS7OOy7SsINzU7nqN4pAGz+9bRW25541xsMu+VV3l1XEo/SREQc4fo97ubWFpeF+rhjse7uKSQleAM3dnwGy56CjBzI6B/4nd4v8DutD3i8zhQtIke89uxxu7aPO5oR/Xuw6d5pzP9kO7f8exVtvS/999MF/GXWOIwxsG87rH0FKlu4SJbxBkO8X2SwZ/SH9P4Hbqdmgcf1H2REpBs77Pa4WzLmV2+yt7Iu5vZr7ziL5No9UF4c/NkZOV2xK/C7as/BD/YkBIM8POCbwj0s8FN7g2n5eiwicuRpzx73ERHcAI8u2sDsSUMZdstrMbVfeMNEFn9RyunHZHPcgMgx4Us27WHn/mq+fkJ2MMSL8ZftxJbvZH/JdjIb92CC8ynfCdUtXBDLmxjcg+8ffe89oz+kZCrgRY4ACu5W1DX4efaTbVxxam7EkMFYzJ40NGL44QdzzmJgrxR2V9Ry04KVvL02cNBz8vBsLhs3mPNH9Q80rK+BimIo3xW29x78XRG2J1+z/+CVepMO7p5Jb95d0w+SeyngRVxMwR0jv9/y8Dvr+d1b65k4rA/vrd/dqcu/+5LjyT+qFxnJCfTrkcxHG/dwxrHZEZestdby2fZ9jBmcCfXVLXTPhHXNNN1XW3bwyhKSA0Gelg2J6ZCUHvidmBb8yQibToOkjOj36SCsSNwpuA/B/775BQ+9vd7x9ZySm8mGkgq+qgp8Jdsj3z6JicOy6ZGcEDhQ2syitSWMHtSTrPQkqKs8OOAritlfsh1/RSmZCXWBNrUVUFcRmG6sjb04X2qMgd/szSHafb5UHbAVaYOC+xA1daFcOmYQz31aFJd19k5LDB1AnXZCf15dVYwxcM7Ifry5eleo3cOXncQpuZn4vB7SkxJI9nnZvreKjOQE8u96E4At900Pta9taMTvhxSv/0CINwv12uoykhqrwu4rPzBdVwl15S08rgL8DTH+dSYy/Ju/ISQ1C/mm375USExtfdqrE6rk8KDg7kR+v2VPZR0Pv72evy7ZGpo/Z+oI7nttbRdWFt3mX0/DGENJWQ3j7n0bgFEDenDVhDye+nAz4/OyuPWC4yivqeeZpdu499W1zPvvr4WuvrhldyVri8t44PV1vHrdRKyFZJ+Hc377H351yfGMGZxJXaOfJNPA125/ga+P6snt5+dG7uHXVeKvLcdTf/AbxcYvd9EvuYFUarB1FXjD3yBs1K81bZnHFxnobYZ9CvjSordpfjshsZP/OyItU3A74NZ/r+JvS7YBB4KxeH8Ny7d/xf7qej7evJd/ffolt04fyX9PHMrm3ZWc+eC7ocf/4vzhvLRiR1y+/OHCEwfw0oodrbZ588ZJnPu/iyPm3XXxKG574fOIeaMH9WRl0X4uHz849Pf365HErrJalt16Diff/Vao7Vs/PYNj+qbzysqd/OjvnwLwvdNymTUhj6N6p2CMYWNpBWc3u0RB6BOCtdBQC/VV/Pjp91mzrZiF156Cz18NdVVQH/ZT13y6MnCM4KD5TT/VgTcH287vLPUktP2G4Etp+VNCYlrwjSJ8Ok1vDNIiBbcD1u8qZ/rD7/PwZflMOT6nxTabd1eS1yetzWU99p+N3PfaWtbcNYWURG+7R7e41bnHRXb7NPfQzHy27qniurOHhbbJ6zdMYnj/DACK99ewr7qOPulJ9ElPYuueSgb2SiHBG+g/31BSwTm//Q9/vuoUJg3Lxhs8CGytDRw3sBYa66C+iim/eZ0BqY3M/c7xUFdFXU0FNVXl9PDUQ33gtqmv4sM125kwOIUEf03ojaCiopx124vJ75eIt7GayooyvI01+Pw1eG2s3UdBnoSwME+JMh3tDaGttmngPezOsTtsKbhd5otd5ZwX3Pt99bqJTHs4cMr+z849lmP7Z3DGsdkk+46cgH/+2tP4+u8/DN3O65NGSVkNlXUH9pafnT2ebz2xJHT73z+awCWPfhCxnKsm5HJM33Rueb6Qodlp/HDSUG56btVB63vlutO55flClm/fx4rbz+OzbV/xvT9/Err/9GP6UFJew0/PPZbTjunD6DveiFq7jwbW334GVZVlvL1yC9OH98DTWBPY22/xk0LTdGXoTcHUV5HkryGhsfrgTxXt7EqynkRMYowh70sJ/CQkgy8ZElIO/E5ICrsveLv5/RqOekgU3C6UO+cVrj97GDeeeyyFX+6nV6qPQZmpB7Vp0iM5gbKayL27hTdMjLhOyw8nDeXxKJe9DXfF+CER/fdHuqauoI7act907n11DU8s3sSsCXm8vHIHJeW1vPfLM/n70m089cEWPrvtXBK9HjbtruTllTv4+kkD6ZWayIl3HnhTWHnHeaE3iQSPocHvp3+qhzE5iXy2cQcpppYUarnqlL68VLCBZGo5My+NVVt2kkwtqdSSamqZNa4f/166gXOHpZPpq8fWVlFVWU66JzD6qKy8HE9DFWmmFkPH88AmJENCMqalYPcl0+hNYmVxLflD+2MSWnqDSA57Y0iOuL/Bk4jHl4InMSXy/sNotJKC+zBVUl7DuHveZv09U/F5Pby2aifXzPuUZ2eP5+QhmSR4PVz912Us/LwYiBxdEm1v/fxR/Xj8isBz5Yo/fRwxlv2ZH4ynuKyaG59dAcCkY7NZ/MXB13H54u6pHHtrbGekHgmenjWOK+cu7eoyYvLDM4by+H+a3twtq26dTIa3IXCsoaE6cPJYQzU01FJXU8k1T33AL84awrDePh5/+3OOzUrgnGE9oKGWx97+nGTqOHVwKsOzEtm6aw+bdu5mZHYiNNSwd18ZSdSRZOrpn2IDn0QaavC097hDmEbjo5pEqvw+Mntk4PP5At1Pxhs4H8HjDZtumu8Jm04ItvGETTfN9zRr46WwuJLSynrOHJFzYL7Hix8vHm+0dXjZU9VIVkZKszq8EeszQycpuI8UJeU19M1IDt1uaPRz9ytr+MGkoQzslRKa//763az6cj/Lt3/FOSP78YsFK/nr98dF/YKJUL9ws9t1DX4eeWc9D7+zIXTflvumR3T3NPf0rHGMHZKJMfDaqmIuyh/AAwvX8uR7mzl1aBYfbdrT4sFSib+zRvTlnbUl9M1IYs7UEfz0HyvIzkjiwW+cGPFm9Kcrx/L9pwOv8VkT8lhbXMaHGw9cu+ehmflcP395TOv00kgydaGfJFNPMnX87crR/OgvH5FEHcnUR9zXvG3T9OlDM+mXnhAYqmr94G+gsbGBhoYGkjx+rL8R628MvFn4G8HfwJod+/DgJ90HA3sm4vc3Yhsb8OBnX0U1GYkGY/1YfwP19fV48JPo8WPaOwKqDebOMgW3OKtpD/4X5w/nR2ceA8Cqov18ua+aq/+2LKJt+J5/rErKami0lkfe2cC8jwOjWfKP6kX+Ub146sMtoXZ3XjSK2188MBLml1OGYy385vV1LS73L7PG8d1D3Bu+6+JRDOiZwhcl5TywMLCeLfdNP2KOQbjBNZOP5uozjqZnii/0f/nklnP49pNLWF9Sweq7zue4214/6HHXnz2sHSfgWR657ER++swyPPhJoBEPfrz4ScAfmvaaxsDviJ/IeecM78PVV31PwS3O2r63ipdW7uDaycccdN8HG3aT4DF864kljB2SyYJrTuvUdYcH5IZ7pnLMLa9x4YkDeOhb+aHLCYy6bSGVdY08PWsc+6rqmH5CDrUNftKSEvjbkq2MGZzJ+pJyzjuuP5t3V/LH9zbxvQm5XPRI4ADntZOPZv4n20MnRf3/C47jVy+v5qGZ+VycPzCilqOz03j7Z5N5/rOiULdSa/pmJFFSHtmHfv+lJ7R44PTFH08I1fTY5WO4f+E6Nu+ubHG57/3yTCY+sKjN9Uv3tPX+CzovuI0xycBiIInA9bsXWGtvb+0xCm4BKPxyP8P6pR/4oopO8t25S1n8RSlLbj6b/j2TKa+pJzUxITT8D2B/dT3lNfUHHeBtL2stxWU15PRMYV9VHb1SWx933dT+hvnLufK0XI7KTOXCR94P3T/1+P78/PzhEWPZN907DY/HRLwhnXhUL/r3SOLxK8ZS1+Bn2davOPXoLAA2llbQr0cyb3xezPTROSzdvJdGv2Xy8L4UbNnLG6t3Mff9zTT4LbdMG8m8j7dy6tF9eGbpNh64dDS/fG4lEDgfIe/mVwEixukfiovzB/DC8tbPIQiX6PVQ1+hncO9Utu2titpu1oQ8ThjUI6Y3Rrfq7OA2QJq1tsIY4wPeB6631i6J9hgFtzjJ77dU1jWQkeyO0933V9Wzq7yGfj2S6Zniw++3zHr6E/ZV1fPPq0/FFxyH/uKKHbzxeTGPfHvMIa/zrpdWM/eDzXxx91QSEyJHXny5r5peKT7SkhKY9MAitu2tovDO8zn+9oO7DgCG98tg3a6DTxy76+JRXJw/MDQSZsM9U0nweiLegNbdPYXhty4E4MFvnMilYwYy7+NtJHo9jB+aRf+eyTyzdBuXjx9CaXkt43/9NjNOHsSCZUW887MzOKp3KrvKakJvwDX1jZxy91uU10aOqHrwGyfy8392n1AfM7gXn27bB0BWWiLTTshpceRWZqovdL2iTg3uiMbGpBII7mustR9Ha6fgFulajX5LdX0j6Umtn4DT9PpvOhDdFLqzJw1lf1U9zxZsZ8t906mua2TkbQtZcPWp/OAvBcyffWroxKj6Rj8JHhNaxva9VUx8YBE/OvNofnH+CH7wlwLy+qTx/6aN7LS/b1XRfi585H2W33Zu6FPQg6+v45FFG9p4ZKQN90xlX3U9Y8POAA43cVgfhvfL4OPNe1n1ZeCyyz+cNJSd+2tYsmkPJeW1fPtrg7nnkuPZub+GPRV1nDCoZ4vLqm1o5LF3N/GNsYM47b53gMjzNjo9uI0xXmAZcAzwqLX2ptbaK7hF3Kmh0c+8j7fx3VOHtHiVyu6svtHP3Pc307dHUqhLZe2vplBaXhvq+286UF7b0IjXmNBZt4vWlnDiUb3omeLjvtfWcMX4XAZnRXazLd++j0GZKfRJTzrkWnPnvML00Tk8+u0x3PyvlTyzdLuje9y9gOeBn1hrC5vdNxuYDTB48OCTt27VCR0i0jU2lFQwJCs11A3VndXUN3LXy6v59X+Ndm5UiTHmdqDSWvtgtDba4xYRaZ/2nDnZ5tuRMSY7uKeNMSYFOAfontczFRE5AsRy6bAc4OlgP7cH+Ie19mVnyxIRkWjaDG5r7UrgpDjUIiIiMej+PfciIhJBwS0i4jIKbhERl1Fwi4i4jIJbRMRlFNwiIi6j4BYRcRkFt4iIyyi4RURcRsEtIuIyCm4REZdRcIuIuIyCW0TEZRTcIiIuo+AWEXEZBbeIiMsouEVEXEbBLSLiMgpuERGXUXCLiLiMgltExGUU3CIiLqPgFhFxmTaD2xhzlDFmkTFmjTHmc2PM9fEoTEREWpYQQ5sG4GfW2k+NMRnAMmPMm9ba1Q7XJiIiLWhzj9tau9Na+2lwuhxYAwx0ujAREWlZu/q4jTG5wEnAx04UIyIibYs5uI0x6cBzwA3W2rIW7p9tjCkwxhSUlpZ2Zo0iIhImpuA2xvgIhPY8a+2/WmpjrX3CWjvWWjs2Ozu7M2sUEZEwsYwqMcCfgDXW2t86X5KIiLQmlj3uCcAVwFnGmOXBn2kO1yUiIlG0ORzQWvs+YOJQi4iIxEBnToqIuIyCW0TEZRTcIiIuo+AWEXEZBbeIiMsouEVEXEbBLSLiMgpuERGXUXCLiLiMgltExGUU3CIiLqPgFhFxGQW3iIjLKLhFRFxGwS0i4jIKbhERl1Fwi4i4jIJbRMRlFNwiIi6j4BYRcRkFt4iIyyi4RURcRsEtIuIyCm4REZdpM7iNMXONMSXGmMJ4FCQiIq2LZY/7KWCKw3WIiEiM2gxua+1iYG8cahERkRh0Wh+3MWa2MabAGFNQWlraWYsVEZFmOi24rbVPWGvHWmvHZmdnd9ZiRUSkGY0qERFxGQW3iIjLxDIc8BngI2C4MabIGPN958sSEZFoEtpqYK29LB6FiIhIbNRVIiLiMgpuERGXUXCLiLiMgltExGUU3CIiLqPgFhFxGQW3iIjLKLhFRFxGwS0i4jIKbhERl1Fwi4i4jIJbRMRlFNwiIi6j4BYRcRkFt4iIyyi4RURcRsEtIuIyCm4REZdRcIuIuIyCW0TEZRTcIiIuo+AWEXEZBbeIiMvEFNzGmCnGmHXGmA3GmDlOFyUiItG1GdzGGC/wKDAVOA64zBhznNOFiYhIy2LZ4x4HbLDWbrLW1gHzgYudLUtERKKJJbgHAtvDbhcF54mISBdIiKGNaWGePaiRMbOB2cGbtcaYwkMpzEF9gN1dXUQrVF/HdefaQPUdiu5cG3ROfUNibRhLcBcBR4XdHgTsaN7IWvsE8ASAMabAWjs21iLiqTvXBqrvUHTn2kD1HYruXBvEv75Yuko+AYYZY/KMMYnATOBFZ8sSEZFo2tzjttY2GGN+DLwOeIG51trPHa9MRERaFEtXCdbaV4FX27HcJzpWTlx059pA9R2K7lwbqL5D0Z1rgzjXZ6w96DijiIh0YzrlXUTEbay1nfYDTAHWARuAOZ257GbrOQpYBKwBPgeuD86/A/gSWB78mRb2mJuDda0Dzm+rZiAP+BhYDzwLJLazxi3AqmAdBcF5vYE3g8t8E8gMzjfAw8EaVgJjwpZzZbD9euDKsPknB5e/IfhY047ahodto+VAGXBDV24/YC5QAhSGzXN8e0VbRwy1/QZYG1z/80Cv4PxcoDpsGz7W0Rpa+ztjqM/x/yWQFLy9IXjhNmW7AAAE1ElEQVR/bjvqezasti3A8q7YfkTPkm7x3Iv6emlvaLbywvMCG4GhQCKwAjius5bfbF05TRsMyAC+IHA6/h3Az1tof1ywnqTgk3BjsN6oNQP/AGYGpx8DrmlnjVuAPs3mPdD0ggDmAPcHp6cBrwWfFOOBj8P+sZuCvzOD001PoKXAqcHHvAZMPYT/WzGBMaRdtv2AScAYIl/cjm+vaOuIobbzgITg9P1hteWGt2u2nHbVEO3vjLE+x/+XwLUEg5XAaLNnY62v2f3/A9zWFduP6FnSLZ57UV8vHXmxR9mopwKvh92+Gbi5s5bfxrpfAM5t5ckaUQuBETKnRqs5uIF3c+CFGdEuxpq2cHBwrwNywp4w64LTjwOXNW8HXAY8Hjb/8eC8HGBt2PyIdu2s8zzgg+B0l24/mr1o47G9oq2jrdqa3fd1YF5r7TpSQ7S/M8Zt5/j/sumxwemEYLsWP/m1sl0MgTOzh3Xl9gtr05Ql3ea519JPZ/Zxd8mp8caYXOAkAh/VAH5sjFlpjJlrjMlso7Zo87OAfdbahmbz28MCbxhjlgXPKgXoZ63dCRD83beD9Q0MTjef3xEzgWfCbneX7Qfx2V7R1tEeswjsSTXJM8Z8Zoz5jzFmYljN7a3hUF9TTv8vQ48J3r8/2L49JgK7rLXrw+Z1yfZrliXd+rnXmcEd06nxnckYkw48B9xgrS0D/gAcDeQDOwl8BGuttvbOb48J1toxBK6q+CNjzKRW2nZFfQRPqLoI+GdwVnfafq3pNvUYY24BGoB5wVk7gcHW2pOAnwJ/N8b06GANh1J3PP6XnbFdLyNyx6FLtl8LWdLeZcb1udeZwR3TqfGdxRjjI7Ch51lr/wVgrd1lrW201vqBJwlc2bC12qLN3w30MsYkNJsfM2vtjuDvEgIHr8YBu4wxOcH6cwgcsOlIfUXB6ebz22sq8Km1dlew1m6z/YLisb2iraNNxpgrgQuA79jg511rba21dk9wehmBfuNjO1hDh19Tcfpfhh4TvL8nsDeW+sIe818EDlQ21R337ddSlnRgmXF97nVmcMft1HhjjAH+BKyx1v42bH5OWLOvA00XunoRmGmMSTLG5AHDCBwwaLHm4ItwETAj+PgrCfR9xVpfmjEmo2maQD9yYbCOK1tY5ovAd03AeGB/8KPT68B5xpjM4Efd8wj0L+4Eyo0x44Pb4rvtqS9MxN5Od9l+YeKxvaKto1XGmCnATcBF1tqqsPnZwWvYY4wZSmBbbepgDdH+zljqi8f/MrzuGcA7TW9gMTqHQP9vqCsh3tsvWpZ0YJlxe+4BnT4ccBqBo7IbgVs6c9nN1nM6gY8bKwkb7gT8lcCwm5XBjZIT9phbgnWtI2wERrSaCRxdX0pgCM8/gaR21DeUwFH5FQSGGN0SnJ8FvE1g+M/bQG974ADNo8EaVgFjw5Y1K1jDBuCqsPljCbwYNwKP0I7hgMHHpwJ7gJ5h87ps+xF4A9kJ1BPYS/l+PLZXtHXEUNsGAn2aEcPWgEuD//MVwKfAhR2tobW/M4b6HP9fAsnB2xuC9w+Ntb7g/KeAq5u1jev2I3qWdIvnXrQfnTkpIuIyOnNSRMRlFNwiIi6j4BYRcRkFt4iIyyi4RURcRsEtIuIyCm4REZdRcIuIuMz/ATca36ncfyaTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(7, 2e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('model-30k-vocab')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model perplexity is exp(validation loss):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24.36271768396201"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.exp(3.193054)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('model-30k-vocab')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}