train_with_word2vec_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12597, 300)\n",
      "[-0.09085934 -0.04050202 -0.07604051 -0.02878256 -0.03832901  0.04651292\n",
      " -0.08233552 -0.06436986 -0.006646    0.01952864 -0.10288478 -0.03497215\n",
      " -0.04002167 -0.0277764  -0.03175622  0.0141541   0.06412307  0.0514068\n",
      " -0.04425988 -0.01241343 -0.00785599 -0.0206115   0.03097875 -0.01636746\n",
      "  0.12936752 -0.04187576 -0.04594978  0.0632828  -0.0185187  -0.03435634\n",
      "  0.02050968 -0.00153008  0.04422459  0.08578489  0.0569248  -0.13749051\n",
      "  0.07906641 -0.08986761 -0.06780145  0.03066873 -0.07235949  0.00491482\n",
      " -0.05130845 -0.03616726  0.02364809  0.00438806  0.03820136 -0.02138964\n",
      "  0.01468734  0.0239164   0.06650317 -0.01117458  0.08711758  0.02350685\n",
      "  0.00737275 -0.03050523  0.01972778 -0.00599776  0.00697179  0.03140137\n",
      "  0.01172278 -0.00411805 -0.09804209 -0.06642748 -0.01673794  0.04739327\n",
      " -0.00381328 -0.10510307 -0.06244999 -0.03497938 -0.02515736 -0.05637315\n",
      " -0.03300777 -0.02991769 -0.00337767  0.01365327  0.03197937 -0.01513318\n",
      " -0.00577635 -0.00223164 -0.04746583  0.02690253 -0.01221982  0.00873462\n",
      "  0.02593678 -0.01298416  0.03521911 -0.01153397 -0.0180837   0.19437511\n",
      "  0.07214894  0.0191956   0.01476469 -0.12907593 -0.01212172  0.03786235\n",
      "  0.0190078   0.02728468  0.02551379  0.00874735  0.02364352 -0.05328384\n",
      "  0.04863269  0.12145241 -0.01912275  0.00469045  0.0606837  -0.05783218\n",
      " -0.00574705  0.06287996 -0.01838772  0.04039203 -0.05672992 -0.01512217\n",
      " -0.00257998  0.03880896 -0.05094773 -0.02234471 -0.0101133   0.06988217\n",
      " -0.07711124 -0.04651542 -0.01086473  0.12196324 -0.06704399  0.00697516\n",
      " -0.048647   -0.0259877   0.07690135  0.06530562 -0.09058807  0.08880168\n",
      " -0.03158594 -0.02760033  0.02793602 -0.02565172  0.06985817  0.10143013\n",
      "  0.07203253 -0.02400441 -0.01870823  0.02618405  0.06419723  0.06780434\n",
      "  0.02404639 -0.01851569  0.00811303  0.01423165  0.02782483  0.05012083\n",
      "  0.05280148 -0.09563148  0.0434398   0.02146245 -0.01855817 -0.0173738\n",
      "  0.07151752 -0.03521181 -0.02936102  0.01273108  0.04211192  0.00988951\n",
      "  0.11457489  0.03447227  0.03735168 -0.09684919  0.00257806 -0.09837919\n",
      " -0.00097418 -0.01557509 -0.04510913  0.08372375 -0.01635526  0.10140619\n",
      " -0.13361511 -0.02750501 -0.08441713  0.01672048 -0.00660872  0.00829647\n",
      " -0.0281041   0.01076135  0.00046972 -0.06391156 -0.05648893 -0.0850139\n",
      "  0.06795205  0.10212228 -0.02036593 -0.01175962  0.00721332 -0.00237448\n",
      "  0.11135048 -0.09262906  0.09503172  0.01378813  0.11511543 -0.02581553\n",
      "  0.01192933  0.02188368 -0.01705039  0.07703491  0.02277727  0.00582705\n",
      " -0.07017894 -0.07617793  0.04443178 -0.0521546   0.0353219   0.05361279\n",
      " -0.02656942  0.01532776 -0.03446891 -0.01847338 -0.07524516  0.05025817\n",
      " -0.09387384 -0.05220094  0.00103038  0.04862991 -0.15522934 -0.05111457\n",
      "  0.09838269 -0.02504948  0.05110523  0.1032243  -0.06817696 -0.04459864\n",
      " -0.01139168 -0.0697358  -0.01057485 -0.01302617  0.03544144  0.01298461\n",
      " -0.04073605  0.08345529  0.07624066  0.09410781 -0.01756834  0.08591017\n",
      "  0.0392493  -0.03339985  0.06212961 -0.11667953 -0.02196945 -0.00998627\n",
      "  0.03804541  0.01198692 -0.09078339  0.00712901 -0.0563237  -0.05370563\n",
      "  0.08309761  0.05206729 -0.04540391  0.02631334 -0.00429622 -0.11247702\n",
      " -0.10659108  0.008087   -0.07673101 -0.06894144 -0.06808041  0.00093352\n",
      " -0.1679322  -0.02717449  0.03608102 -0.07517123  0.00114856  0.02856596\n",
      " -0.02516653 -0.02587932 -0.10417227 -0.05486893  0.01422467  0.10518625\n",
      "  0.14748152 -0.02123361  0.145761    0.06357961 -0.01562561  0.06651055\n",
      " -0.04852806 -0.05579144  0.01395309 -0.03206687 -0.00442388 -0.08798324\n",
      "  0.01627165 -0.08187773  0.06859025  0.02167492 -0.01108429  0.03047448\n",
      "  0.13988569  0.03800683  0.04920618 -0.02150288  0.00588878  0.06126343]\n",
      "kitchen\n",
      "restaurant\n",
      "[(u'delish', 0.7754588723182678), (u'tasty', 0.7434055805206299), (u'yummy', 0.7149284482002258), (u'scrumptious', 0.6782612800598145), (u'flavorful', 0.6213445663452148), (u'divine', 0.6012349724769592), (u'tastey', 0.5901238918304443), (u'delicous', 0.587270200252533), (u'mouthwatering', 0.5798896551132202), (u'devine', 0.5537630319595337)]\n",
      "[(u'mexican', 0.7112151384353638), (u'cantonese', 0.6801303029060364), (u'asian', 0.6524285078048706), (u'japanese', 0.6484655141830444), (u'americanized', 0.6477989554405212), (u'indian', 0.6398634910583496), (u'filipino', 0.6290761232376099), (u'vietnamese', 0.6171532869338989), (u'korean', 0.6015596389770508), (u'lebanese', 0.5581501722335815)]\n",
      "(399850, 100)\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "train cnn mode for sentiment classification on yelp data set\n",
    "author: hao peng\n",
    "'''\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from Word2VecUtility import Word2VecUtility\n",
    "from gensim.models import word2vec\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation, Flatten\n",
    "# from keras.layers.embeddings import Embedding\n",
    "from keras.layers.convolutional import Convolution1D, MaxPooling1D\n",
    "\n",
    "data = pd.read_csv('review_sub_399850.tsv', header=0, delimiter=\"\\t\", quoting=3, encoding='utf-8')\n",
    "model = word2vec.Word2Vec.load(\"300features_40minwords_10context\")\n",
    "print model.syn0.shape\n",
    "print model[\"chinese\"]\n",
    "print model.doesnt_match(\"man woman child kitchen\".split())\n",
    "print model.doesnt_match(\"coffee tea juice restaurant\".split())\n",
    "print model.most_similar(\"delicious\")\n",
    "print model.most_similar(\"chinese\")\n",
    "\n",
    "# data embedding parameters\n",
    "max_length = 100\n",
    "# max_words = 5000\n",
    "# max_words = model.syn0.shape[0]\n",
    "num_features = 300\n",
    "\n",
    "# model training parameters\n",
    "batch_size = 32\n",
    "# embedding_dims = 100\n",
    "nb_filter = 250\n",
    "filter_length = 3\n",
    "hidden_dims = 250\n",
    "nb_epoch = 2\n",
    "\n",
    "# index trick parameters\n",
    "index_from = 3\n",
    "# padding = 0\n",
    "start = 1\n",
    "oov = 2\n",
    "\n",
    "words_set = set(model.index2word)\n",
    "word2index = { word : (i + index_from) for i,word in enumerate(words_set) }\n",
    "index2word = { i : word for word, i in word2index.items() }\n",
    "index2word[0] = '0'\n",
    "index2word[1] = '1'\n",
    "index2word[2] = '2'\n",
    "# 'Word2Vec' object does not support item assignment\n",
    "padding_model = {}\n",
    "padding_model['0'] = np.random.standard_normal(num_features)\n",
    "padding_model['1'] = np.random.standard_normal(num_features)\n",
    "padding_model['2'] = np.random.standard_normal(num_features)\n",
    "\n",
    "\n",
    "reviews_words = []\n",
    "for review in data[\"text\"]:\n",
    "    review_words = Word2VecUtility.review_to_wordlist(review, remove_stopwords = True)\n",
    "    # each word index increased with 3.\n",
    "    review_words = [start] + [word2index[w] if (w in words_set) else oov for w in review_words]\n",
    "#   review_words = [oov if (ix > (max_words + index_from)) else ix for ix in review_words]\n",
    "    reviews_words.append(review_words)\n",
    "\n",
    "# padding with 0, each review has max_length now.\n",
    "reviews_words = sequence.pad_sequences(reviews_words, maxlen = max_length, padding='post', truncating='post')\n",
    "print reviews_words.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(399850, 100, 300)\n"
     ]
    }
   ],
   "source": [
    "data_matrix = np.empty((reviews_words.shape[0], max_length, num_features))\n",
    "print data_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[    1  2903  1184  4192  4272  8238  9911   619  2446  4470  6337  7137]\n",
      " [    1   159  8305  2446  8305  5323  6822  1591  6674  3217  1153  6124]\n",
      " [    1   438 10258  7152  4718  4679  4645  3813  2342 11460  4345  6058]\n",
      " [    1  4099  7972  9232 10792 10659  8279 10439 10850  7486  7972   755]\n",
      " [    1   509  4281  4501  6639   853   159   904  4281  4501  2446  2764]\n",
      " [    1 10456  8828  4035  5285   448 11832  4924  6689  5375  1066  5132]\n",
      " [    1  1066   438 10654  5091  3316  3956     2  4501  9584  8907  6337]\n",
      " [    1  2980   965  2342  9790  5604  5695  8763  1381  4603  9319  7693]\n",
      " [    1  4129  6424   159 10772  4807  2425  4501     2  5025  7805  9915]\n",
      " [    1  1623  4170  9815  3969  8831  5577  4158 10650 11240  4977  9254]\n",
      " [    1  9077  5323  1184   651  8336  2903  1255  6545  5323  6124  8694]\n",
      " [    1 11496  1042  5323  2901  2758   395 11005  7170  7362  3354  7192]\n",
      " [    1   280  9378  8828     2  1091  6500 10154  3771    12 11067  9312]\n",
      " [    1  8452  4967     0     0     0     0     0     0     0     0     0]\n",
      " [    1  5643  1648     2  4218     2  9406  8213  2687     2  6330   965]\n",
      " [    1  2903  5092  5758  8561     2  9480   755  5776  2424  9584  5101]\n",
      " [    1  5323  8114   159  7486  7297   615  2063  4950     0     0     0]\n",
      " [    1 12254   367  3233  3771  5629  6254   618  7074  4874  1015     0]\n",
      " [    1  2406     2  3457  2446  4501  1648  4648  7374 12016  5731  2406]\n",
      " [    1  8011  6243  9565  6412  7360  6500  1372 11627  7297     2   755]]\n"
     ]
    }
   ],
   "source": [
    "print reviews_words[:20, :12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.39257071  1.36350982  1.10130961 ..., -0.32868679 -0.10299574\n",
      "  -0.98516773]\n",
      " [-0.07528156  0.0281019  -0.12271535 ...,  0.02202644 -0.05857712\n",
      "   0.07863296]\n",
      " [-0.02730094  0.01866139 -0.09900852 ...,  0.04338717  0.1285717\n",
      "   0.02086206]\n",
      " ..., \n",
      " [ 1.44873275  1.5635236   3.13083889 ...,  0.04700205  0.65883317\n",
      "  -1.29727728]\n",
      " [ 1.44873275  1.5635236   3.13083889 ...,  0.04700205  0.65883317\n",
      "  -1.29727728]\n",
      " [ 1.44873275  1.5635236   3.13083889 ...,  0.04700205  0.65883317\n",
      "  -1.29727728]]\n"
     ]
    }
   ],
   "source": [
    "# print ([index2word[ix] for ix in reviews_words[0]])\n",
    "print np.array([model[index2word[ix]] if (index2word[ix] in model) else padding_model[index2word[ix]] for ix in reviews_words[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_matrix = np.empty((reviews_words.shape[0], max_length, num_features))\n",
    "for i in xrange(0, reviews_words.shape[0]):\n",
    "    data_matrix[i,:,:] = np.array([model[index2word[ix]] if (index2word[ix] in model) else padding_model[index2word[ix]] for ix in reviews_words[0]])\n",
    "\n",
    "# del(reviews_words)\n",
    "\n",
    "labels = data[\"stars\"]\n",
    "print labels[:10], labels.shape\n",
    "labels[labels <= 3] = 0\n",
    "labels[labels > 3] = 1\n",
    "print labels[:10]\n",
    "print (labels == 0).sum()\n",
    "\n",
    "index = np.arange(data_matrix.shape[0])\n",
    "train_index, valid_index = train_test_split(index, train_size = 0.8, random_state = 520)\n",
    "del(index, data)\n",
    "\n",
    "train_data = data_matrix[train_index]\n",
    "valid_data = data_matrix[valid_index]\n",
    "train_labels = labels[train_index]\n",
    "valid_labels = labels[valid_index]\n",
    "\n",
    "print train_data.shape\n",
    "print valid_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print train_data.shape\n",
    "print valid_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print \"start training model...\"\n",
    "\n",
    "# model = Sequential()\n",
    "\n",
    "# we start off with an efficient embedding layer which maps\n",
    "# our vocab indices into embedding_dims dimensions\n",
    "# model.add(Embedding(max_features, embedding_dims, input_length=maxlen))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "# we add a Convolution1D, which will learn nb_filter\n",
    "# word group filters of size filter_length:\n",
    "\n",
    "# filter_length is like filter size, subsample_length is like step in 2D CNN.\n",
    "model.add(Convolution1D(nb_filter=nb_filter,\n",
    "                        filter_length=filter_length,\n",
    "                        border_mode='valid',\n",
    "                        activation='relu',\n",
    "                        subsample_length=1))\n",
    "# we use standard max pooling (halving the output of the previous layer):\n",
    "model.add(MaxPooling1D(pool_length=2))\n",
    "\n",
    "# We flatten the output of the conv layer,\n",
    "# so that we can add a vanilla dense layer:\n",
    "model.add(Flatten())\n",
    "\n",
    "# We add a vanilla hidden layer:\n",
    "model.add(Dense(hidden_dims))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "# We project onto a single unit output layer, and squash it with a sigmoid:\n",
    "model.add(Dense(1))\n",
    "model.add(Activation('sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              class_mode='binary')\n",
    "model.fit(train_data, train_labels, batch_size=batch_size,\n",
    "          nb_epoch=nb_epoch, show_accuracy=True,\n",
    "          validation_data=(valid_data, valid_labels))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}