E2_word_pronounciations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Problem\n",
    "\n",
    "Build a model to help pronounce english words. We'll convert english words in to [Arpabet](https://en.wikipedia.org/wiki/Arpabet) phoneme\n",
    "\n",
    "@sunilmallya\n",
    "\n",
    "https://www.twitch.tv/videos/171226133"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Dataset\n",
    "\n",
    "http://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Load data\n",
    "\n",
    "#cmudict.0.6d\n",
    "\n",
    "data = open('cmudict-0.7b', 'r').readlines()\n",
    "phones = open('cmudict-0.7b.phones', 'r').readlines()\n",
    "phones = open('cmudict-0.7b.symbols', 'r').readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "words = []\n",
    "phones = []\n",
    "\n",
    "def f_char(word):\n",
    "    for c in [\"(\", \".\", \"'\", \")\", \"-\", \"_\", \"\\xc0\", \"\\xc9\"]:\n",
    "        #print c in word, type(word)\n",
    "        if c in word:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "for d in data:\n",
    "    parts = d.strip('\\n').split('  ')   \n",
    "    if not f_char(parts[0]):\n",
    "        words.append(parts[0])\n",
    "        phones.append(parts[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['A', 'A42128', 'AA', 'AAA', 'AABERG'],\n",
       " ['AH0',\n",
       "  'EY1 F AO1 R T UW1 W AH1 N T UW1 EY1 T',\n",
       "  'EY2 EY1',\n",
       "  'T R IH2 P AH0 L EY1',\n",
       "  'AA1 B ER0 G'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[:5], phones[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(116519, 116519)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words), len(phones)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set(['IH2', 'IH0', 'IH1', 'AH2', 'AH0', 'AH1', 'JH', 'EH2', 'EH0', 'EH1', 'EY1', 'EY0', 'EY2', '4', '8', 'AY1', 'AY0', 'AY2', 'D', 'H', 'AW2', 'AW1', 'AW0', 'P', 'T', 'AO1', 'AO0', 'X', 'OY2', 'OY1', 'OY0', 'UW2', 'UW1', 'UW0', 'HH', 'UH2', 'UH0', 'UH1', '3', '7', 'C', 'ZH', 'G', 'K', 'O', 'S', 'W', 'AE1', 'AE0', 'AE2', '0', 'NG', '2', '6', 'B', 'F', 'J', 'N', 'R', 'V', 'Z', 'SH', 'DH', 'CH', '1', '5', 'TH', '9', 'AA1', 'AA0', 'A', 'E', 'I', 'AA2', 'M', 'L', 'Q', 'U', 'Y', 'OW1', 'OW0', 'OW2', 'ER0', 'ER1', 'ER2', 'IY1', 'IY0', 'IY2', 'AO2'])\n"
     ]
    }
   ],
   "source": [
    "all_chars = set()\n",
    "for word, phone in zip(words, phones):\n",
    "    for c in word:\n",
    "        all_chars.add(c)\n",
    "    for p in phone.split(\" \"):\n",
    "        all_chars.add(p)\n",
    "        \n",
    "print all_chars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets define some helper functions to convert words to symbols and vice versa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['IH2', 'IH0', 'IH1', 'AH2', 'AH0', 'AH1', 'JH', 'EH2', 'EH0', 'EH1', 'EY1', 'EY0', 'EY2', '4', '8', 'AY1', 'AY0', 'AY2', 'D', 'H', 'AW2', 'AW1', 'AW0', 'P', 'T', 'AO1', 'AO0', 'X', 'OY2', 'OY1', 'OY0', 'UW2', 'UW1', 'UW0', 'HH', 'UH2', 'UH0', 'UH1', '3', '7', 'C', 'ZH', 'G', 'K', 'O', 'S', 'W', 'AE1', 'AE0', 'AE2', '0', 'NG', '2', '6', 'B', 'F', 'J', 'N', 'R', 'V', 'Z', 'SH', 'DH', 'CH', '1', '5', 'TH', '9', 'AA1', 'AA0', 'A', 'E', 'I', 'AA2', 'M', 'L', 'Q', 'U', 'Y', 'OW1', 'OW0', 'OW2', 'ER0', 'ER1', 'ER2', 'IY1', 'IY0', 'IY2', 'AO2', '+']\n"
     ]
    }
   ],
   "source": [
    "# Create a map of symbols to numbers\n",
    "symbol_set = list(all_chars)\n",
    "symbol_set.append(\"+\") # add space for padding\n",
    "\n",
    "# word to symbol index\n",
    "def word_to_symbol_index(word):\n",
    "    return [symbol_set.index(char) for char in word]\n",
    "\n",
    "# list of symbol index to word\n",
    "def symbol_index_to_word(indices):\n",
    "    return [symbol_set[idx] for idx in indices]\n",
    "\n",
    "# phone to symbol index\n",
    "def phone_to_symbol_index(phone):\n",
    "    return [symbol_set.index(p) for p in phone.split(\" \")]\n",
    "\n",
    "# list of symbol index to word\n",
    "def psymbol_index_to_word(indices):\n",
    "    return [symbol_set[idx] for idx in indices]\n",
    "\n",
    "print symbol_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[70, 58, 18, 54, 71, 58, 42] ['A', 'R', 'D', 'B', 'E', 'R', 'G']\n"
     ]
    }
   ],
   "source": [
    "# sample\n",
    "\n",
    "indices = word_to_symbol_index(\"ARDBERG\")\n",
    "print indices, symbol_index_to_word(indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[68, 54, 82, 42] ['AA1', 'B', 'ER0', 'G']\n"
     ]
    }
   ],
   "source": [
    "indices = phone_to_symbol_index(\"AA1 B ER0 G\")\n",
    "print indices, symbol_index_to_word(indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(34, 32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pad input and output data\n",
    "\n",
    "input_sequence_length = max([len(w) for w in words])\n",
    "output_sequence_length = max([len(p.split(' ')) for p in phones])\n",
    "\n",
    "input_sequence_length, output_sequence_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# input data\n",
    "trainX = []\n",
    "labels = []\n",
    "\n",
    "def pad_string(word, max_len, pad_char = \"+\"):\n",
    "    out = ''\n",
    "    for _ in range(max_len - len(word)):\n",
    "        out += pad_char\n",
    "        \n",
    "    return out + word\n",
    "    \n",
    "#for word in words:\n",
    "#    padded_strng = \"%*s\"  % (input_sequence_length, word)\n",
    "#    trainX.append(word_to_symbol_index(padded_strng))\n",
    "\n",
    "# output data\n",
    "#for p in phones:\n",
    "#    padded_strng = \"%*s\"  % (output_sequence_length, p)\n",
    "#    print phone_to_symbol_index(padded_strng)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'+++++++++++++++++++++++++EY2 EY1'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pad_string('EY2 EY1', output_sequence_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "for word in words:\n",
    "    padded_strng = pad_string(word, input_sequence_length)\n",
    "    trainX.append(word_to_symbol_index(padded_strng))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# output labels\n",
    "# TODO: Fix padding logic\n",
    "\n",
    "labels =[]\n",
    "for p in phones:\n",
    "    label = []\n",
    "    for _ in range(output_sequence_length - len(p.split(' '))):\n",
    "        label.append(phone_to_symbol_index('+')[0])\n",
    "    label.extend(phone_to_symbol_index(p))\n",
    "    labels.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(116519, 116519)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(labels), len(trainX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INP:  ['+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', 'A', 'A']\n",
      "LBL:  ['+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', '+', 'EY2', 'EY1']\n"
     ]
    }
   ],
   "source": [
    "trainX[0], labels[0]\n",
    "\n",
    "print \"INP: \", symbol_index_to_word(trainX[2])\n",
    "print \"LBL: \", symbol_index_to_word(labels[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare the data for modeling and build iterators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(104867, 34) (104867, 32)\n",
      "(11652, 34) (11652, 32)\n",
      "[DataDesc[data,(128, 34L),<type 'numpy.float32'>,NCHW]] [DataDesc[target,(128, 32L),<type 'numpy.float32'>,NCHW]]\n"
     ]
    }
   ],
   "source": [
    "import mxnet as mx\n",
    "import numpy as np\n",
    "\n",
    "def shuffle_together(a, b):\n",
    "    assert len(a) == len(b)\n",
    "    p = np.random.permutation(len(a))\n",
    "    return a[p], b[p]\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "trainX, labels = np.array(trainX), np.array(labels)\n",
    "trainX, labels = shuffle_together(trainX, labels)\n",
    "\n",
    "N = int(len(trainX) * 0.9) # 90%\n",
    "\n",
    "dataX = np.array(trainX)[:N]\n",
    "dataY = np.array(labels)[:N]\n",
    "\n",
    "testX = np.array(trainX)[N:]\n",
    "testY = np.array(labels)[N:]\n",
    "\n",
    "print dataX.shape, dataY.shape\n",
    "print testX.shape, testY.shape\n",
    "\n",
    "\n",
    "## Lets define the Iterator\n",
    "train_iter = mx.io.NDArrayIter(data=dataX, label=dataY,\n",
    "                                 data_name=\"data\", label_name=\"target\",\n",
    "                                 batch_size=batch_size,\n",
    "                                 shuffle=True)\n",
    "\n",
    "test_iter = mx.io.NDArrayIter(data=testX, label=testY,\n",
    "                                 data_name=\"data\", label_name=\"target\",\n",
    "                                 batch_size=batch_size,\n",
    "                                 shuffle=True)\n",
    "\n",
    "print train_iter.provide_data, train_iter.provide_label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model - 2 Layer LSTM (Encoder - Decoder or Seq2Seq model) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data_dim = len(symbol_set)\n",
    "\n",
    "data = mx.sym.var('data')  # Shape: (N, T)\n",
    "target = mx.sym.var('target')  # Shape: (N, T)\n",
    "\n",
    "# 2 Layer LSTM \n",
    "# get_next_state = return the states that can be used as starting states next time\n",
    "lstm1 = mx.rnn.FusedRNNCell(num_hidden=128, prefix=\"lstm1_\", get_next_state=True)\n",
    "lstm2 = mx.rnn.FusedRNNCell(num_hidden=128, prefix=\"lstm2_\", get_next_state=False)\n",
    "\n",
    "# In the layout, 'N' represents batch size, 'T' represents sequence length,\n",
    "# and 'C' represents the number of dimensions in hidden states.\n",
    "  \n",
    "# one hot encode \n",
    "data_one_hot = mx.sym.one_hot(data, depth=data_dim) # Shape: (N, T, C)\n",
    "data_one_hot = mx.sym.transpose(data_one_hot, axes=(1, 0, 2)) # Shape: (T, N, C)\n",
    "\n",
    "# Note that when unrolling, if 'merge_outputs'== True, the 'outputs' is merged into a single symbol\n",
    "# encoder  (with repeat vector)\n",
    "_, encode_state = lstm1.unroll(length=input_sequence_length, inputs=data_one_hot, layout=\"TNC\")\n",
    "encode_state_h = mx.sym.broadcast_to(encode_state[0], shape=(output_sequence_length, 0, 0)) #Shape: (T,N,C); use ouput seq shape\n",
    "\n",
    "# decoder\n",
    "decode_out, _ = lstm2.unroll(length=output_sequence_length, inputs=encode_state_h, layout=\"TNC\")\n",
    "decode_out = mx.sym.reshape(decode_out, shape=(-1, batch_size))\n",
    "\n",
    "# logits out\n",
    "logits = mx.sym.FullyConnected(decode_out, num_hidden=data_dim, name=\"logits\")\n",
    "logits = mx.sym.reshape(logits, shape=(output_sequence_length, -1, data_dim))\n",
    "logits = mx.sym.transpose(logits, axes=(1, 0, 2))\n",
    "\n",
    "# Lets define a loss function: Convert Logits to softmax probabilities\n",
    "loss = mx.sym.mean(-mx.sym.pick(mx.sym.log_softmax(logits), target, axis=-1))\n",
    "loss = mx.sym.make_loss(loss)\n",
    "\n",
    "# visualize\n",
    "#shape = {\"data\" : (batch_size, dataX[0].shape[0])}\n",
    "#mx.viz.plot_network(loss, shape=shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "net = mx.mod.Module(symbol=loss,\n",
    "                    data_names=['data'],\n",
    "                    label_names=['target'],\n",
    "                    context=mx.gpu())\n",
    "\n",
    "net.bind(data_shapes=train_iter.provide_data,\n",
    "            label_shapes=train_iter.provide_label)\n",
    "\n",
    "\n",
    "net.init_params(initializer=mx.init.Xavier())\n",
    "net.init_optimizer(optimizer=\"adam\",\n",
    "                   optimizer_params={'learning_rate': 1E-3,\n",
    "                                     'rescale_grad': 1.0},\n",
    "                   kvstore=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets also have a test net to track our progress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# lets keep a test network to see how we do \n",
    "predict_net = mx.mod.Module(symbol=logits,\n",
    "                         data_names=['data'],\n",
    "                         label_names=None,\n",
    "                         context=mx.gpu())\n",
    "\n",
    "data_desc = train_iter.provide_data[0]\n",
    "\n",
    "# shared_module = True: sharesthe same parameters and memory of the training network\n",
    "predict_net.bind(data_shapes=[data_desc],\n",
    "              label_shapes=None,\n",
    "              for_training=False,\n",
    "              grad_req='null',\n",
    "              shared_module=net)\n",
    "\n",
    "def predict(data_iter):\n",
    "    data_iter.reset()\n",
    "    corr = 0\n",
    "    for i, data_batch in enumerate(data_iter):\n",
    "        #print data_batch.label[0]\n",
    "        predict_net.forward(data_batch=data_batch)\n",
    "        predictions = predict_net.get_outputs()[0].asnumpy()\n",
    "        indices = np.argmax(predictions, axis=2)\n",
    "        lbls = data_batch.label[0].asnumpy()\n",
    "        results = (indices == lbls)\n",
    "        for r in results:\n",
    "            # Exact match\n",
    "            if np.sum(r) == output_sequence_length:\n",
    "                corr += 1.0\n",
    "            \n",
    "            # total % match per sample\n",
    "            #corr += (1.0 *np.sum(r)/ output_sequence_length)\n",
    "    return corr/data_iter.num_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Epoch:', '0001', 'cost =', '0.084552024')\n",
      "('Epoch:', '0001', 'test acc =', '0.453055269')\n",
      "('Epoch:', '0002', 'cost =', '0.081247869')\n",
      "('Epoch:', '0002', 'test acc =', '0.460264332')\n",
      "('Epoch:', '0003', 'cost =', '0.080448788')\n",
      "('Epoch:', '0003', 'test acc =', '0.463783042')\n",
      "('Epoch:', '0004', 'cost =', '0.079517347')\n",
      "('Epoch:', '0004', 'test acc =', '0.443872297')\n",
      "('Epoch:', '0005', 'cost =', '0.079031543')\n",
      "('Epoch:', '0005', 'test acc =', '0.440611054')\n",
      "('Epoch:', '0006', 'cost =', '0.080047944')\n",
      "('Epoch:', '0006', 'test acc =', '0.413491246')\n",
      "('Epoch:', '0007', 'cost =', '0.078811273')\n",
      "('Epoch:', '0007', 'test acc =', '0.460693443')\n",
      "('Epoch:', '0008', 'cost =', '0.079148563')\n",
      "('Epoch:', '0008', 'test acc =', '0.453741847')\n",
      "('Epoch:', '0009', 'cost =', '0.080474575')\n",
      "('Epoch:', '0009', 'test acc =', '0.435547545')\n",
      "('Epoch:', '0010', 'cost =', '0.080831558')\n",
      "('Epoch:', '0010', 'test acc =', '0.441469276')\n",
      "('Epoch:', '0011', 'cost =', '0.080418202')\n",
      "('Epoch:', '0011', 'test acc =', '0.458719533')\n",
      "('Epoch:', '0012', 'cost =', '0.082743175')\n",
      "('Epoch:', '0012', 'test acc =', '0.461723309')\n",
      "('Epoch:', '0013', 'cost =', '0.082254162')\n",
      "('Epoch:', '0013', 'test acc =', '0.459749399')\n",
      "('Epoch:', '0014', 'cost =', '0.079607155')\n",
      "('Epoch:', '0014', 'test acc =', '0.457432200')\n",
      "('Epoch:', '0015', 'cost =', '0.078253003')\n",
      "('Epoch:', '0015', 'test acc =', '0.463439753')\n",
      "('Epoch:', '0016', 'cost =', '0.078124710')\n",
      "('Epoch:', '0016', 'test acc =', '0.460607621')\n",
      "('Epoch:', '0017', 'cost =', '0.079100017')\n",
      "('Epoch:', '0017', 'test acc =', '0.457947134')\n",
      "('Epoch:', '0018', 'cost =', '0.080882906')\n",
      "('Epoch:', '0018', 'test acc =', '0.463868864')\n",
      "('Epoch:', '0019', 'cost =', '0.078724205')\n",
      "('Epoch:', '0019', 'test acc =', '0.461294198')\n",
      "('Epoch:', '0020', 'cost =', '0.077563618')\n",
      "('Epoch:', '0020', 'test acc =', '0.443014075')\n",
      "('Epoch:', '0021', 'cost =', '0.076945196')\n",
      "('Epoch:', '0021', 'test acc =', '0.461723309')\n",
      "('Epoch:', '0022', 'cost =', '0.073986541')\n",
      "('Epoch:', '0022', 'test acc =', '0.468074150')\n",
      "('Epoch:', '0023', 'cost =', '0.073073949')\n",
      "('Epoch:', '0023', 'test acc =', '0.463182286')\n",
      "('Epoch:', '0024', 'cost =', '0.072131226')\n",
      "('Epoch:', '0024', 'test acc =', '0.463096464')\n",
      "('Epoch:', '0025', 'cost =', '0.072078408')\n",
      "('Epoch:', '0025', 'test acc =', '0.461980776')\n"
     ]
    }
   ],
   "source": [
    "epochs = 125\n",
    "total_batches =  len(dataX) // batch_size\n",
    "for epoch in range(epochs):\n",
    "    avg_loss = 0\n",
    "    train_iter.reset()\n",
    "    for i, data_batch in enumerate(train_iter): \n",
    "        net.forward_backward(data_batch=data_batch)\n",
    "        loss = net.get_outputs()[0].asscalar()\n",
    "        avg_loss += loss /total_batches\n",
    "        net.update()\n",
    "    \n",
    "    # every 10 epochs\n",
    "    test_acc = predict(test_iter)\n",
    "    \n",
    "    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_loss))\n",
    "    print('Epoch:', '%04d' % (epoch + 1), 'test acc =', '{:.9f}'.format(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Save the model\n",
    "prefix = 'pronounce128';\n",
    "net.save_checkpoint(prefix, epochs)\n",
    "#pred_model = mx.mod.Module.load(prefix, num_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "expression predicted actual\n",
      "     SUNIL S UW1 N IH1 L\n",
      "    JOSEPH JH OW1 S AH0 F\n",
      "   RANDALL R AE1 N D AH0 L\n",
      " SAUSALITO S AO2 S AH0 L IY1 T OW0\n",
      "EMBARCADERO EH0 M B AA0 R AH0 K EH1 R OW0\n",
      "      KAJA K AA1 JH AH0\n",
      "    AMULYA AE2 M Y UW1 L Y AH0\n",
      "    TWITCH T W IH1 CH\n",
      "  ALUMINUM AH0 L UW1 M AH0 N AH0 M\n"
     ]
    }
   ],
   "source": [
    "# Test module\n",
    "\n",
    "test_net = mx.mod.Module(symbol=logits,\n",
    "                         data_names=['data'],\n",
    "                         label_names=None,\n",
    "                         context=mx.gpu())\n",
    "\n",
    "data_desc = train_iter.provide_data[0]\n",
    "\n",
    "# shared_module = True: sharesthe same parameters and memory of the training network\n",
    "test_net.bind(data_shapes=[data_desc],\n",
    "              label_shapes=None,\n",
    "              for_training=False,\n",
    "              grad_req='null',\n",
    "              shared_module=net)\n",
    "\n",
    "def print_word(arr):\n",
    "    word_indices = symbol_index_to_word(arr)\n",
    "    out = filter(lambda x: x != symbol_set[-1], word_indices)\n",
    "    return \"\".join(out)\n",
    "\n",
    "def print_phone(arr):\n",
    "    word_indices = psymbol_index_to_word(arr)\n",
    "    out = filter(lambda x: x != symbol_set[-1], word_indices)\n",
    "    return \" \".join(out)\n",
    "\n",
    "testX, testY = trainX[0:10], labels[0:10]\n",
    "#print testX\n",
    "testX = [word_to_symbol_index(pad_string(\"SUNIL\", input_sequence_length))]\n",
    "testX += [word_to_symbol_index(pad_string(\"JOSEPH\", input_sequence_length))]\n",
    "testX += [word_to_symbol_index(pad_string(\"RANDALL\", input_sequence_length))]\n",
    "testX += [word_to_symbol_index(pad_string(\"SAUSALITO\", input_sequence_length))]\n",
    "testX += [word_to_symbol_index(pad_string(\"EMBARCADERO\", input_sequence_length))]\n",
    "testX += [word_to_symbol_index(pad_string(\"KAJA\", input_sequence_length))]\n",
    "testX += [word_to_symbol_index(pad_string(\"AMULYA\", input_sequence_length))]\n",
    "testX += [word_to_symbol_index(pad_string(\"TWITCH\", input_sequence_length))]\n",
    "testX += [word_to_symbol_index(pad_string(\"ALUMINUM\", input_sequence_length))]\n",
    "\n",
    "testX = np.array(testX, dtype=np.int)\n",
    "\n",
    "test_net.reshape(data_shapes=[mx.io.DataDesc('data', (1, input_sequence_length))])\n",
    "predictions = test_net.predict(mx.io.NDArrayIter(testX, batch_size=1)).asnumpy()\n",
    "\n",
    "print \"expression\", \"predicted\", \"actual\"\n",
    "\n",
    "for i, prediction in enumerate(predictions):\n",
    "    #x_str = symbol_index_to_word(testX[i])\n",
    "    word = print_word(testX[i])\n",
    "    index = np.argmax(prediction, axis=1)\n",
    "    result = print_phone(index)\n",
    "    #result = [symbol_set[j] for j in index]\n",
    "    \n",
    "    print \"%10s\" % word, result\n",
    "    #label = [alphabet[j] for j in testY[i]]\n",
    "    #print \"\".join(x_str), \"\".join(result), \"    \", \"\".join(label)\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO: Convert Arapabet to IPA and call Polly API "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}