Wiki Parsing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xml.etree.cElementTree as ET\n",
    "from IPython.display import clear_output\n",
    "from ipywidgets import FloatProgress, IntText\n",
    "from IPython.display import display\n",
    "from nltk.tokenize.stanford import StanfordTokenizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "import nltk\n",
    "import time\n",
    "import re\n",
    "import pandas as pd\n",
    "import collections\n",
    "from math import log \n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#nltk.download()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting Articles from Specific Categories\n",
    "\n",
    "This is my first attempt at processing the Wikipedia dump.  It streamingly parses the Wikipedia XML and processes any article with a category tag that contains the specified strings.  Currently, it just writes the body of the articles to both a single file and a category specific file.  It probably should also do the tokenization, but doesn't yet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ArticleProcessor:\n",
    "    \n",
    "    def __init__(self, categories):\n",
    "        self.categories = categories        \n",
    "        self.global_matcher = re.compile(\"\\[\\[Category:[^\\]]*(\" + \n",
    "                                         \"|\".join(categories) + \n",
    "                                         \")[^\\]]*\", re.IGNORECASE)\n",
    "        self.category_matcher = {}\n",
    "        self.article_writer = {}\n",
    "        self.global_writer = open(\"data/all-articles\", \"w\")\n",
    "        for category in self.categories:\n",
    "            self.category_matcher[category] = re.compile(\"\\[\\[Category:[^\\]]*\" + \n",
    "                                                         category + \n",
    "                                                         \"[^\\]]*\", re.IGNORECASE)\n",
    "            self.article_writer[category] = open(\"data/\" + category + \"-articles\", \"w\")\n",
    "            \n",
    "    def is_article_of_interest(self, article_text):\n",
    "        return self.global_matcher.search(article_text)\n",
    "\n",
    "    def process_article(self, article_text):\n",
    "        self.global_writer.write(article_text)\n",
    "        self.global_writer.write(\"\\n\")\n",
    "        for category in self.categories:\n",
    "            if self.category_matcher[category].search(article_text):\n",
    "                self.article_writer[category].write(article_text)\n",
    "                self.article_writer[category].write(\"\\n\")\n",
    "    \n",
    "    def close_all(self):\n",
    "        self.global_writer.close()\n",
    "        for writer in self.article_writer.values():\n",
    "            writer.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d400b5563d145e8bc9973d3d331497b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3c1ec1d3c62421eaf29cf924126f38c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4c65b3e106148d6a4193acf90e6f695",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Articles: 17773690 Time: 5338.332955360413 seconds\n"
     ]
    }
   ],
   "source": [
    "#Don't run this\n",
    "\n",
    "p = ET.iterparse(\"data/enwiki-20170820-pages-articles.xml\", \n",
    "                 events=(\"start\", \"end\"))\n",
    "\n",
    "start = time.time()\n",
    "article_count = 0\n",
    "root = None\n",
    "f = FloatProgress(min=0, max=17773690)\n",
    "t = IntText(value=0, description=\"Articles\")\n",
    "m = IntText(value=0, description=\"Matching Articles\")\n",
    "display(t, m, f)\n",
    "\n",
    "processor = ArticleProcessor([\"sportspeople\",\n",
    "                              \"artists\",\n",
    "                              \"politicians\",\n",
    "                              \"military personnel\",\n",
    "                              \"scientist\",\n",
    "                              #sportmanager\n",
    "                              #cleric\n",
    "                              \"monarch\",\n",
    "                              \"Fictional\\ characters\",\n",
    "                              \"nobility\",\n",
    "                              \"criminals\",\n",
    "                              \"judges\"\n",
    "                              \n",
    "                             ])\n",
    "try:\n",
    "    \n",
    "    for event, elem in p:\n",
    "        if root == None:\n",
    "            root = elem\n",
    "        if event == \"end\" and elem.tag == '{http://www.mediawiki.org/xml/export-0.10/}text':\n",
    "            article_count += 1\n",
    "            if article_count % 1000 == 0:\n",
    "                f.value = article_count\n",
    "                t.value = article_count\n",
    "            if elem.text and processor.is_article_of_interest(elem.text):\n",
    "                m.value += 1 \n",
    "                processor.process_article(elem.text)\n",
    "            root.clear()\n",
    "finally:\n",
    "    processor.close_all()\n",
    "    print(\"Articles:\", article_count, \"Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Extracting Articles with Gender Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wiki id</th>\n",
       "      <th>gender</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>307</td>\n",
       "      <td>MALE</td>\n",
       "      <td>Abraham Lincoln</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>339</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>Ayn Rand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>340</td>\n",
       "      <td>MALE</td>\n",
       "      <td>Alain Connes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>344</td>\n",
       "      <td>MALE</td>\n",
       "      <td>Allan Dwan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>595</td>\n",
       "      <td>MALE</td>\n",
       "      <td>Andre Agassi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>628</td>\n",
       "      <td>MALE</td>\n",
       "      <td>Aldous Huxley</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>676</td>\n",
       "      <td>MALE</td>\n",
       "      <td>Andrei Tarkovsky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>700</td>\n",
       "      <td>MALE</td>\n",
       "      <td>Arthur Schopenhauer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>711</td>\n",
       "      <td>MALE</td>\n",
       "      <td>Albert Sidney Johnston</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>736</td>\n",
       "      <td>MALE</td>\n",
       "      <td>Albert Einstein</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   wiki id  gender                    name\n",
       "0      307    MALE         Abraham Lincoln\n",
       "1      339  FEMALE                Ayn Rand\n",
       "2      340    MALE            Alain Connes\n",
       "3      344    MALE              Allan Dwan\n",
       "4      595    MALE            Andre Agassi\n",
       "5      628    MALE           Aldous Huxley\n",
       "6      676    MALE        Andrei Tarkovsky\n",
       "7      700    MALE     Arthur Schopenhauer\n",
       "8      711    MALE  Albert Sidney Johnston\n",
       "9      736    MALE         Albert Einstein"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gender_label_table = pd.read_csv(\"data/wiki.genders.txt\", sep='\\t')\n",
    "gender_label_table.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "862171"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wiki_ids_with_gender = set([str(x) for x in gender_label_table[\"wiki id\"]])\n",
    "len(gender_label_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "is_male = {}\n",
    "for _, row in gender_label_table.iterrows():\n",
    "    is_male[str(row[\"wiki id\"])] = (row[\"gender\"] == 'MALE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_male[\"700\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e92857b6359346d39e2724c1ef3b516f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>IntText</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "IntText(value=0, description='Articles')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d47269160d964169b24c344292ab9be2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>IntText</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "IntText(value=0, description='Matching Articles')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28670ef7b9e5476bbb3cf2dd1b1a1918",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>FloatProgress</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "FloatProgress(value=0.0, max=17773690.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Articles: 17773690 Time: 1406.9675903320312 seconds\n"
     ]
    }
   ],
   "source": [
    "p = ET.iterparse(\"data/enwiki-20170820-pages-articles.xml\", \n",
    "                 events=(\"start\", \"end\"))\n",
    "\n",
    "start = time.time()\n",
    "article_count = 0\n",
    "root = None\n",
    "f = FloatProgress(min=0, max=17773690)\n",
    "t = IntText(value=0, description=\"Articles\")\n",
    "m = IntText(value=0, description=\"Matching Articles\")\n",
    "display(t, m, f)\n",
    "\n",
    "article = None\n",
    "id = None\n",
    "\n",
    "article_writer = open(\"data/gendered-labeled-articles\", \"w\")\n",
    "\n",
    "try:\n",
    "    is_current_article_labeled = False\n",
    "    for event, elem in p:\n",
    "        if root == None:\n",
    "            root = elem\n",
    "        if event == \"start\" and elem.tag == '{http://www.mediawiki.org/xml/export-0.10/}page':\n",
    "            id = None\n",
    "        if id == None and event == \"end\" and elem.tag == '{http://www.mediawiki.org/xml/export-0.10/}id':\n",
    "            is_current_article_labeled = (elem.text in wiki_ids_with_gender)\n",
    "            id = elem.text\n",
    "        if event == \"end\" and elem.tag == '{http://www.mediawiki.org/xml/export-0.10/}text':\n",
    "            article_count += 1\n",
    "            if article_count % 1000 == 0:\n",
    "                f.value = article_count\n",
    "                t.value = article_count\n",
    "            if is_current_article_labeled and elem.text:\n",
    "                m.value += 1 \n",
    "                article = elem.text\n",
    "                article_writer.write(id)\n",
    "                article_writer.write(' ')\n",
    "                article_writer.write(article.replace('\\n', ' '))\n",
    "                article_writer.write('\\n')\n",
    "            root.clear()\n",
    "finally:\n",
    "    article_writer.close()\n",
    "    print(\"Articles:\", article_count, \"Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once this completes, we have a file where each line starts with an article ID and is followed by the full markdown text of the article.  For speed, I'm cleaning it up with this command line chain of seds:\n",
    "\n",
    "```\n",
    "cat gendered-labeled-articles | sed 's/[^a-zA-Z0-9]/ /g' | \\\n",
    "sed 's/\\s\\{2,\\}/ /g' | \\\n",
    "tr '[:upper:]' '[:lower:]' > \\\n",
    "gendered-labeled-articles.stripped\n",
    "```\n",
    "\n",
    "That takes about 17.5 minutes to run.  This reduces the file from 5.8GB to 5.1GB.\n",
    "\n",
    "# Creating a Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "688b287e830a45d599d4e290c4dd3717",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5b40b93957d4548bdb030f6bff151aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Articles: 850399 Time: 430.8352212905884 seconds\n"
     ]
    }
   ],
   "source": [
    "#Don't run this\n",
    "\n",
    "fp = FloatProgress(min=0, max=850399)\n",
    "uw = IntText(value=0, description=\"Unique\")\n",
    "display(fp, uw)\n",
    "article_count = 0\n",
    "word_count = 0\n",
    "\n",
    "\n",
    "article_ids = set()\n",
    "vocab_mapping = {}\n",
    "vocab_writer = open(\"data/vocab\", \"w\")\n",
    "\n",
    "start = time.time()\n",
    "try:\n",
    "    with open(\"data/gendered-labeled-articles.stripped\", 'r') as f:\n",
    "        for line in f:\n",
    "            article_count += 1\n",
    "            if article_count % 1000 == 0:\n",
    "                fp.value = article_count\n",
    "                uw.value = word_count\n",
    "            words = line.split()\n",
    "            article_ids.add(words[0])\n",
    "            for word in words[1:]:\n",
    "                val = vocab_mapping.get(word, -1)\n",
    "                if val == -1:\n",
    "                    word_count += 1\n",
    "                    vocab_mapping[word] = word_count\n",
    "                    vocab_writer.write(word + \" \" + str(word_count) + \"\\n\")\n",
    "\n",
    "finally:        \n",
    "    vocab_writer.close()\n",
    "    print(\"Articles:\", article_count, \"Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7869208"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e373c163ed94de2b1149a23777abae9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>FloatProgress</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "FloatProgress(value=0.0, max=850399.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c09392b6374c4e7187fe4657d31a3dcd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>IntText</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "IntText(value=0, description='Unique')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7404abae16a54c398bfc3bdbb248d40d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>IntText</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "IntText(value=0, description='Unique')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fp = FloatProgress(min=0, max=850399)\n",
    "male_articles = IntText(value=0, description=\"Unique\")\n",
    "female_articles = IntText(value=0, description=\"Unique\")\n",
    "display(fp, male_articles, female_articles)\n",
    "article_count = 0\n",
    "\n",
    "male_counter = collections.Counter()\n",
    "female_counter = collections.Counter()\n",
    "\n",
    "start = time.time()\n",
    "try:\n",
    "    with open(\"data/gendered-labeled-articles.stripped\", 'r') as f:\n",
    "        for line in f:\n",
    "            article_count += 1\n",
    "            if article_count % 1000 == 0:\n",
    "                fp.value = article_count\n",
    "            words = line.split()\n",
    "            if is_male[words[0]]:\n",
    "                male_articles.value += 1\n",
    "                male_counter.update(words[1:])\n",
    "            else:\n",
    "                female_articles.value += 1\n",
    "                female_counter.update(words[1:])\n",
    "finally:    \n",
    "    print(\"Articles:\", article_count, \"Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_counter = male_counter + female_counter\n",
    "all_counter.most_common(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class PMICalculator:\n",
    "    \n",
    "    def __init__(self, counter1, counter2, n1, n2):\n",
    "        self.counter1 = counter1\n",
    "        self.counter2 = counter2\n",
    "        self.n1 = n1\n",
    "        self.n2 = n2\n",
    "        self.lpc1 = log(n1 / (n1 + n2))\n",
    "\n",
    "    def pmi(self, word):\n",
    "        try:\n",
    "            return (log(self.counter1[word]) - \n",
    "            self.lpc1 -\n",
    "            log(self.counter1[word] + self.counter2[word]))\n",
    "        except ValueError:\n",
    "            return 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pc = PMICalculator(female_counter, male_counter, 130576, 718413)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pc.pmi(\"his\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pc.pmi(\"she\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pc.pmi(\"president\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_words = all_counter.most_common(5000)\n",
    "top_words[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_words_with_pmi = [(p[0], p[1], pc.pmi(p[0])) for p in top_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_words_with_pmi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pmi_per_year = [(i, pc.pmi(str(i))) for i in range(1900, 2015)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEKCAYAAADuEgmxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xu4XHV97/H3JyFBthAlO4EHxOxt\nrfoUW6yyRfFSTxUUaQW81Gp3YhT7pEC1WC9HfNIbbfFQbD2GqgdzFE9McoqX2koriki9tB4vBFGu\nIsGyIUolFzVcWhKS7/ljrXGvTNZas2bPrLntz+t51jMza9bM/FZmZ33n9/v+LooIzMzMqljQ7wKY\nmdnwcNAwM7PKHDTMzKwyBw0zM6vMQcPMzCpz0DAzs8ocNMzMrDIHDTMzq8xBw8zMKjuk3wXotmXL\nlsXk5GS/i2FmNlSuv/76HRGxvNVxIxc0Jicn2bJlS7+LYWY2VCTNVDnOzVNmZlaZg4aZmVXmoGFm\nZpU5aJiZWWUOGmZmVpmDhpnZMNm8GSYnYcGC5Hbz5p5+/Mh1uTUzG1mbN8OaNfDQQ8njmZnkMcD0\ndE+K4JqGmdmwWLt2NmA0PPRQsr9HHDTMzIbF3Xe3t78GDhpmZsNixYr8/UuX9izP4aBhZjYsLroI\nxsYO3LdoEdx/f5LfiJjNc9QUOBw0zMyGxfQ0rF8PExMgJbdLlsCePQceV2Oew0HDzGyYTE/DXXfB\n/v3J7a5d+cfVlOdw0DAzG0StxmM0no/If31R/qNDDhpmZv3WHCDOOy/JSxTlKRrjNWYKZjMfG0vy\nHzVQFEWpITU1NRVeT8PMhkbzgD1I8hV51+aJiaRJanKyOGBMTCQBo83BfpKuj4ipVsd5RLiZWT/l\nDdgr+jHfyFMU5SukJKjUyM1TZmZ1qDpHVDsJ64jkvZYuzX++pjxGloOGmVm3ZXMOrcZOtHuhn5mB\n3bth8eID99eYx8hy0DAz67Z25ojKG7DXyt69cMQRB47XWL++J5MWOmiYmXVbUZPTzMzBTVXZAXvt\n2LXrwPEanuXWzGxIZPMXy5Ylv/6LzMzAqlXJMY0A0hiw107g6EH+Io+DhplZJ5rzFzt3Jr/+yzR6\nR83MwMqVSaDZvLl4bqk+5S/yOGiYmXUiL3/Rrp07ZxdTap5b6qMfhcsv70v+Ik9fB/dJOg1YBywE\nPhwRFzc9/1bgd4FHgO3A2RFRMKIl4cF9ZtZTCxYUj6toV2PwXh9UHdzXt5qGpIXAB4CXAscDr5V0\nfNNhNwBTEXEC8Cngkt6W0sysSfP4i6IxE1llOY6sHi6mNFf9bJ46CdgaET+IiD3AFcCZ2QMi4ksR\n0aj3fQM4rsdlNDObDRRSksTOjr/IGzORNTYG55wzm+QuCyB9Sm63o59B43HAPZnH29J9Rd4IfK7W\nEpmZNWueHLC5Kap5zMT4eLJl8w8f/GDS7BQBGzcmzzfrY3K7Hf0MGnnhNrdhUNJKYAp4T8HzayRt\nkbRl+/btXSyimc1bjdrFypWtE93ZMRM7diRb0fiJ6enk+U2bBia53Y5+Bo1twOMzj48DftR8kKRT\ngLXAGRHxcN4bRcT6iJiKiKnly5fXUlgzG2FlU5NXsWBB+8urNi+mNAQBA/o7y+11wJMkPQH4IfAa\n4HeyB0h6OvAh4LSIuK/3RTSzkdc8NfnMDFx2WXs9ovbtm+0yOyQX/7nqW00jIh4B3gRcDdwGfCIi\nbpH055LOSA97D3A48ElJ35F0ZZ+Ka2ajpqz5aS5daGtcl3uQ9HU9jYi4Criqad+fZO6f0vNCmdno\ny1v4qKqJieJmqyHoMtspjwg3s/lnLqO4x8aS5HXZHFFD0GW2Uw4aZja/bN5cPcGdle3dlDdH1JB0\nme2Ug4aZzR+NZql2TUwcmODOTmc+ZF1mO+WgYWbzR1mz1NgYnHtu9RrEkHaZ7ZSDhpnNH2WJ6sbI\n7Xlag6jKQcPMRlPzgL3Nm4sT1dnmp3lag6iqr11uzcxqkTdgb9WqZPyFdOA4jHmSwO4W1zTMbPTk\n5S4agaIROMDNT3PgmoaZjZYqXWoj+rrg0TBzTcPMRkc7XWrnwejtOjhomNlwyUtwN7Qz0nsejN6u\ng5unzGx45CW4s7PLVq09OPk9Z65pmNnwyKtJZGeXLao9jI977EWXtBU0JC2QtKSuwpiZlSqqSTT2\nF80JtW6dx150ScugIen/Sloi6dHArcDtkt5Rf9HMzJoU1SQa++fxnFC9UqWmcXxE7AbOIln7YgWw\nqtZSmZnlKapJnH76bHJ87drkONcqalElEb5I0iKSoPH+iNgraQ7LWpmZdagRANauTZqkVqxIAsaG\nDcXJceuqKjWNDwF3AY8GvippAthdZ6HMzAo1zw111VXlyXHrqpZBIyIujYjHRcTpkZgBfr0HZTMz\nS5SNzWiVHLeuqpIIP1rSRyR9Ln18PLC69pKZ2fxRFhQaYzNmZpLpPxrNT41jWiXHrauqNE/9H+Bq\n4Nj08feBt9RVIDObZ4qCwnnnJQFk5cry5qd5vPRqP1QJGssi4hPAfoCIeATYV2upzGz+KBqwd9ll\n5RMPzswkQWXVKjjssGQAn7vZ1q5K76kHJY0DASDp2cDPai2Vmc0fRbmHaNFJU5oNKjt3JrWLjRsd\nLGpWpabxVuBK4ImSvgZ8DHhzraUys+FXlqfImkvuoXkhJXCPqR6p0nvq28ALgOcAvwc8NSJurLtg\nZjbEWiWvs/JyEmUmJoprIe4xVbvCoCHpFY0NOAN4CvBk4GXpPjOzfK0mFoTZmkg2J1FmbAw2bUrG\nZkxM5B/jHlO1K8tpvKzkuQA+3eWymNmoaDV2onmK80ZOokw2uX3RRQe+HtxjqkcKg0ZEvKGXBTGz\nEbJiRX7PpwhYtgx+8pNkRHfWQw/BwoWwL6dz5sTEgQnuvOlELrrISfAeqDK4792SHpt5fKSkv6y3\nWGY21MryFDt3HhwwGvbtqz7monk6EQeMnqjSe+qlEfHTxoOI+Alwen1FMrOhl52ivB2NMRae2nxg\nVQkaCyUd2ngg6TDg0JLjzcxmawJSteMbNQrXIAZalaCxCbhW0hslnQ1cA2zoxodLOk3S7ZK2Srog\n5/lDJX08ff6bkia78blmVqPm8RlLl7Z+zcKFrlEMiZYjwiPiEkk3AS8CBPxFRFzd6QdLWgh8ADgV\n2AZcJ+nKiLg1c9gbgZ9ExC9Keg3wV8Bvd/rZZlaT5l5RMzOwaBEsXgx79uS/ZmzMAWOIVFojPCI+\nFxFvj4i3dSNgpE4CtkbEDyJiD3AFcGbTMWcyW6v5FPAiqWpd18xqUTbSO298xt69cMQRs3mK8XHP\nEzXECmsakv4tIp4n6X7SeacaTwEREUs6/OzHAfdkHm8DnlV0TEQ8IulnwDiwo8PPNrO5yKtJZFfJ\nKxqfsWsX7PB/21FQWNOIiOelt0dExJLMdkQXAgYkweegj53DMUhaI2mLpC3bt2/vQtHMLFfRSO/z\nz09qHUXTe3ik9sioMk5jY5V9c7ANeHzm8XHAj4qOkXQI8BhgV/MbRcT6iJiKiKnly5d3oWhmlquo\nJrFzZ/E05h6pPVKq5DSemn2QXrxP7MJnXwc8SdITJC0GXkMym27WlcyuEvgq4F8iWs2XbGa1abfG\n4JzFyCmbsPBdaT7jBEm70+1+4MfAZzr94HQxpzeRrAp4G/CJiLhF0p9LOiM97CPAuKStJFO0H9Qt\n18x6qJ0ZaSWPsxhBKvvhLmkB8OGIOLt3RerM1NRUbNmypd/FMBtdmzcnuY2ZmeK5oiCpZdx1V0+L\nZnMn6fqImGp1XOk4jYjYL+lp3SuWmQ29Rs2heZbZLOcxRlaVnMY3JD2z9pKY2fDI60XV4DzGSKuy\nRvivA78naQZ4kNlxGifUWjIzG1xFvagaeQwbWVWCxktrL4WZDZei9TI8HmPkVVkjfKaxkYzEfh7w\nwdpLZmaDo3nqkNNPr77uhY2UKoP7Fks6S9IngHuBU4DLai+ZmQ2GxtQhMzPJiO+ZGdiwAVav9roX\n81DZ3FOnAq8FXgJ8CdgInORlYM3mmaKpQ666yvmLeagsp3E18K/A8yLi3wEkretJqcxscBQlvYv2\n20gra546EfgG8EVJ10h6I7CwN8Uys54qm+68KLntpPe8VDbL7Q0R8c6IeCLwZ8DTgcWSPidpTa8K\naGY1y8tZrFkzGzjypg5x0nveqroI09ci4k0k61u8Dzi51lKZWe8U5SzWrk3uT08nSW4nvY2KQaMh\nIvZHxNVOhpt1WVnzUN2q5Cymp5Ok9/79noRwnmsraJhZDVo1D9XNOQtrg4OGWb+1ah6qm3MW1oZK\nQUPSQknHSlrR2OoumNlQ6aR5qd9dWptzFuPjcNhhsGpV75vKbOBVGRH+ZpKFl64BPptu/1xzucyG\nR17z0qpVyQW4ykW33eahTvMfea9v5Cw2boT//M9k+dbGuaxcCcuWOXhYIiJKN2ArMN7quEHZTjzx\nxDDrqYmJiOQSm7+NjUVs2jR7/KZNyWuk5Pbcc5Njyl6TfW3VY/PkvR4ixsdny1X1PGykAFuiwjW2\nStD4EnBIlTcbhM1Bw3pOKg8aELFwYXLBLbron3vugYGk6OJcdFGfmDj42ObgVCUotDqPvM+xkVA1\naJQu9wog6SPAU0iapR7O1FDeW0/dpzNe7tVq11ju9O67kyakBx5ImnNaGRtLcgV5x7ZaGjW7xGoe\nKekOmz2+eWW9sbHihZMaypZvzfscGxlVl3utkgi/mySfsRg4IrOZzT95+Yvdu2Hx4tavfeih4uBS\nlvTOfmaRBQsOzDkU9chqZd++g3tSZbkb7rzXchGmiLgQQNKjI+LB+otkNsDyLsZ79yY9jg4/PLmw\nS0lAaUfjop83aK5sadWGffuSwALJe8y159XERNLV9vzzDw5w7oZrVOs9dbKkW4Hb0sdPk+RFmGx+\nKroY79qVNC9FJD2QFpbM7SkdvK9x0c/WFhq9nMpqGFnZsR1zqRE0gsL0NOzYAZs2eeoQO0iV5qn3\nkaypsRMgIr4L/FqdhTIbCHldU4suxtnmoenpZJGiomaeolpI46K/eXPSxXXlyuoBo6ER1PIG7OUp\nCwqeOsRyVJ2w8J6mXSWZMrMRUDS1R94yp3BwTSE7YK4djc9plVjPq63AbFCr8vmN5LuDgrWhStC4\nR9JzgEiXfn07aVOV2cgqW61u/fr85qfmqT8av9SLLvB5Fi5snb+YmIBzzime+qNRQ1q1Ktl/7rme\nJsS6pkqX22XAOpK1wQV8ATg/Iir0Mew9d7m1rliwoLgZaWKietdXKM5LNCfMq3SJzXbNzXb9Xbo0\n2bdzZ/77rl6dBLxGN+FG7sIs1bUutxGxIyKmI+LoiDgqIlYOasAwa0tezqKxr+zHVKOHVJ68nEfR\nhIDnnHNwTqGsOam5dpA39QccXPbset5uirIOtexyK+kJwJuByezxEXFGfcUyq1nz4LeZGXjDG5IL\n+J49rV8fkf+LPq/Jp3GBzg4ILPul3zwoD5IuvevWzb1Lrtfzti6p0jz1XeAjwE3Az+vdEfGVeos2\nN26eskra6cpaZmKi+00+zSPOW71vWVNatpxlI85t3qvaPNWypgH8V0Rc2oUy2Shr90LXb9345V3X\nhXh6ur1/uxUrygOgk97WRVV6T62T9KfpIL9nNLbaS2bDo52V59qZ1rvOJVDbGfw2Pj7YvY/yciaN\nnIsH5Vm3tZrREPgfwDbgKyQz3n4J+JcqsyGWvOdSkvms7khvj8w55leBrwO3ADcCv13lvT3LbR8U\nzZw6Pt6/KcBbyXv/RYsiFi/O/8y8GWMHyaCXzwYeXZzl9nvACRFRITtYjaRLgF0RcbGkC9Kg8c6m\nY54MRETcIelY4HrglyLip2Xv7ZxGH1RpU4fiOZnymnmKcg7dbBIq6rLamOm1MQ+Tf6XbPNDNWW6/\nCzy28yId4ExgQ3p/A3BW8wER8f2IuCO9/yPgPmB5l8th3VC1qacosOTlF3qxBGpRl9XGTK8OGGYH\nqRI0jga+J+lqSVc2tg4/9+iIuBcgvT2q7GBJJ5FMzX5nwfNrJG2RtGX79u0dFs3aVnWeoyJ5Qafd\nJVA7UTT6Ozu628yAakHjT4GXA+8G/iazlZL0RUk352xntlNASccAG4E3RETu6i8RsT4ipiJiavly\nV0Z6LjvPUWOg2vh4/rHNg+Kap76Q4JBD8gfQVUk+zyV53otajdmoqJL4IKlt/Ga6HVXlNS3e73bg\nmPT+McDtBcctAb4N/FbV93YifEC0s6xp0brVMLuUapXk7lyT5+0soWo2oqiYCK+ynsargW8BvwW8\nGvimpFd1GKuuBFan91cDn8n53MXAPwAfi4hPdvh51i1Vf8k31z7Gx5OlTi+7LHl+48bZ6SzKRjRH\nzCa/W+UXipqZVq4sL2vRNB+D0qXWbJC0iiokifCjMo+XA9+tEpFK3nMcuJaky+21wNJ0/xTw4fT+\nSmAv8J3M9qut3ts1jRrN9Zd8q1pH3q/85tpGFY1aSdFWVlZ3WbV5ji52ub0pIn4l83hBGjR+peRl\nfeMutzWq0g02b2T42rXVZnktUrWbbZWpQTydhlmubna5/Xzac+r1kl4PfBa4qtMC2oDKNj8tW5Zs\njaaoogtyI2FcNDK86HVVAkY7zURVenE5uW3WkZZzT0XEOyS9EnguyXoa6yPiH2ovmfVe88yv2dXj\nGr2Z8i70EUlQeeCB/JxCY7BcVXMdXJedTbYoUNXRZddsHimsaUh6i6RnSjokIv4+It4aEX/ogDHC\nWk2xHVG8jsTMTPESpY3BcllF7zMxAY88knzWXNZ9aAzY27QpP7l9+un1zWdlNg+UNU8dR7Ji332S\nvizp3ZJ+Q9LSHpXNeq1K001E/lKnZbILDDXGcZQtV9oNeWNHVq+GDRuqTaxoZrmqJMIXk/Rqeg5w\ncrr9NCKOr7947XMivAPdWmMia2yseJbVXk+n3ov5rMyGVDcT4YeRDLJ7TLr9CPhmZ8WzgdTpdCCQ\njMdoXsK0KBA0mpLaWYK0k+nSPfLbrGOFiXBJ64GnAveTBIn/B7w3In7So7JZrzUvS5qd+bWKsbHi\nJUm7IW+J1jVrkvtVPrNosSInx80qK6tprAAOBf4D+CHJmhql05LbCMj++t+xI9kmJvKPbadW0Q2d\nTizokd9mHSsMGhFxGvBM4K/TXW8DrpP0BUkX9qJwNiCKLrbr1rXfvNSJTpuX8pLjXtXOrC2l4zTS\noeU3S/op8LN0+03gJJLZb20+aG626tca4N1oXmp3/W0zO0DZOI0/kHSFpHuAr5IEi9uBV5As12rz\nyVyS1t3m5iWzviuraUwCnwL+MNIFk8z6alBqPGbzWMtxGsPG4zTMzNrXzXEaZoOvk/EbZlZZywkL\nzQZWY0R582SK7Y7fMLPKXNMYFv4lfaDsNOxw8Oy77YzfMLPKqiz3+gpJd0j6maTdku6XtLsXhbNU\n3joVK1cma110EjyGORC1mpEXPD2IWQ2q1DQuAc6IiMdExJKIOCIiltRdMMsoukDu3Fl9ltbmAHHe\nefkLJg1L4KgSEDw9iFnXVQkaP46I22oviRUru0CWNcM0AoUEq1YdGCAuu6yzKTl6rTnoLW0xVMjj\nN8xqUTZh4SvSu1skfRz4R+DhxvMR8emay2YNRSOhG/KCSvPkfs1t/kVdrQexSSdvosJFi2DxYtiz\nZ/a4RjK83RX/zKyyst5TL8vcfwh4ceZxAA4avXLRRQdeNJvlNcNUafOv+l79lncue/cmEyYefrgH\n+pn1UGHQiIg3AEh6bkR8LfucpOfWXTDLaFwIzz//4GnKi5ph5lJjkJJf8ZOTg3UBLjqXXbuSWXjN\nrGeq5DT+tuI+q9P0dHKB3LSp2iyt7dYY8sY5DEpSvOhcBrFWZDbiyiYsPFnS24Dlkt6a2f4MaHOR\naOuaqhMHtrsK3yCPc/BEhWYDo6ymsRg4nKQJ64jMtht4Vf1Fs47krR0xPt7eewxKUtzrYJgNjJYT\nFkqaiIiSrjuDZegnLGxMjVFHcre5FxIkv9gPOyx/SdeJiaQ2Y2Yjr+qEhWVdbt8XEW8B3i/poMgS\nEWd0WEZr1uka2K0UTS0O+cHEzT9m1qSwpiHpxIi4XtIL8p6PiK/UWrI5GuqaxuRk/niMXvzir7OG\nY2YDr2pNo0rz1AuBb0TEHDr9995QB40FC/IH3UlJ4tvMrCbdXE/j9cB3JH1d0iWSXibpyI5LaAdz\n19L6DPPkjGYDpGXQiIjXRcSTgVcC24APANvrLti81G7XUl8Iq8mbJXiQxqGYDZEqU6OvlPQhkvXC\nTwHeDzy/kw+VtFTSNemU69eU1VwkLZH0Q0nv7+Qzh0I7XUt9IawubxqSQRqHYjZEquQ0dgB3ApcB\nX4qIuzr+UOkSYFdEXCzpAuDIiHhnwbHrgOXp8W9q9d5DndNoRz+T5sPGuSKzlrqW04iIZcDZwKOA\niyR9S9LGDst3JrAhvb8BOCvvIEknAkcDX+jw80ZP0cC7QRmQN0icKzLrmirNU0uAFcAEMAk8Buj0\n59nREXEvQHp7VM7nLgD+BnhHhTKukbRF0pbt2+dJusUXwuo8DYlZ11TpPfVvJNOk3wj8dkQ8JSJW\nt3qRpC9KujlnO7Ni2c4DroqIe1odGBHrI2IqIqaWL19e8e2HnC+E1XkaErOuKVtPA4CIOGEubxwR\npxQ9J+nHko6JiHslHQPcl3PYycDzJZ1HMgfWYkkPRMQFcynPyCka3e0LYb7paf/bmHVBy6BRkyuB\n1cDF6e1nmg+IiJ//D5f0emDKAaOJL4Rm1mNVmqfqcDFwqqQ7gFPTx0iakvThPpWpv/LGXMxlHIbH\nbphZjVp2uR02Q9nlNm/22UWLkvb37BrYY2PlbfFFs9i6/d7MWuh47ilJf0uyFniuiPiDuRevPgMf\nNPImBly7Nn/MRZ6ycRgeu2Fmc9Tx1OjAAF95h1TR1OfNo5XLlI3D8NgNM6tZYdCIiA1Fz9kcFU1n\nsXAh7NtX7T3KxmGsWJFf0/DYDTPrkrJFmK4se6EXYZqDol/8+/YluYcqOY2ycRgXXeTFlMysVmW9\np04GjgP+FfhrktHZ2c3aVfSLvzHYLDv47KMfhcsvb29AmgexmVnNyhLhC0m6w74WOAH4LPB3EXFL\n74rXvoFOhLt3k5kNqI4nLIyIfRHx+XTKkGcDW4EvS3pzF8s5v7gmYGZDrnREuKRDgd8gqW1MApcC\nn66/WCOouavtxo0OFmY2dMoS4RuAXwY+B1wYETf3rFTDLG8cBuR3tQUHDjMbKmU5jf3Ag5ldjQMF\nREQsqblsc9LXnEZRzuKww2DnzoOP96A7MxsQHQ/ui4h+zUs1vIrGYRQN3vOgOzMbMmXNU48CzgF+\nkWQtjcsj4pFeFWwotRsEPOjOzIZMWW1iAzAF3AScjsdmtFYUBMbHvWCSmY2EsqBxfESsjIgPAa8C\nnt+jMg2votX01q1zV1szGwllXW73Nu5ExCOSelCcIddqNT0HCTMbcmU1jadJ2p1u9wMnNO5L2t2r\nAg6d6emkR9T+/cltWaDwgklmNmTKek8t7GVB5p2iadLBNRIzG1juVtsvRd1z167tT3nMzCpw0OgX\nL5hkZkPIQaNOZTmLou65HrthZgPMQaOh20npRs5iZgYiZnMWjfct6p7rsRtmNsAcNKD1BX4uWuUs\nuj1NuntimVkPFE5YOKzmNGHh5GT+2tqdTCi4YEESgJpJSXfcbvLiTmbWoY4XYZpX6khK9zJn4Z5Y\nZtYjDhpQzwW+lzkL98Qysx5x0IC5X+CzeYRly5KtkVOA3s035Z5YZtYjDhowt6R0c/J8585kyybS\nofqUIp1wTywz6xEnwqtqXsb1gQfyV+PL6uXKfHnLzDoJbmYVVU2EO2hUkdc7qYo6ekqZmdXAvae6\nKa93UhXOKZjZiOlL0JC0VNI1ku5Ib48sOG6FpC9Iuk3SrZIme1vS1Fx6ITmnYGYjqF81jQuAayPi\nScC16eM8HwPeExG/BJwE3Nej8h2obBnXRvJ8fDzZvDKfmY2wfgWNM0nWICe9Pav5AEnHA4dExDUA\nEfFARMyhjagL8nonZZ/bvx927Ei2untKmZn1Ub+CxtERcS9AentUzjFPBn4q6dOSbpD0Hkn9WRiq\n0SV3fPzA/Tt3dj5HlZnZEKktaEj6oqSbc7YzK77FIcDzgbcDzwR+AXh9wWetkbRF0pbt27d3pfzA\ngYP3iqbk8HQdZjaPFC732qmIOKXoOUk/lnRMRNwr6RjycxXbgBsi4gfpa/4ReDbwkZzPWg+sh6TL\nbTfKn7scaxFP12Fm80S/mqeuBFan91cDn8k55jrgSEnL08cvBG7tQdkS7XSzdddaM5sn+hU0LgZO\nlXQHcGr6GElTkj4MEBH7SJqmrpV0EyDgf/eshFVrD+5aa2bzSG3NU2UiYifwopz9W4DfzTy+Bjih\nh0WbtWJFfpPU+Dgcfrin6zCzeckjwosUTQK4bl1vJiE0MxtADhpFur0cq5nZCOhL89TQmJ52kDAz\ny3BNw8zMKnPQMDOzyhw0zMysMgcNMzOrzEHDzMwqc9AwM7PKHDTMzKwyBw0zM6vMQaNZdg2NyUkv\nsGRmluER4Vl5a2isWZPc98hwMzPXNA6Qt4aGV+YzM/s5B42sojU0vDKfmRngoHGgohX4vDKfmRng\noHGgojU0vDKfmRngoHEgr6FhZlbKvaeaeQ0NM7NCrmmYmVllDhpmZlaZg4aZmVXmoGFmZpU5aJiZ\nWWWKiH6XoaskbQdmOniLZcCOLhVnUPichscontconhOM3nlNRMTyVgeNXNDolKQtETHV73J0k89p\neIzieY3iOcHonlcrbp4yM7PKHDTMzKwyB42Dre93AWrgcxoeo3heo3hOMLrnVco5DTMzq8w1DTMz\nq2zkg4akyyXdJ+nmzL6nSfq6pJsk/ZOkJZnn3iVpq6TbJb0ks/+0dN9WSRf0+jyatXNekk6VdH26\n/3pJL8y85sR0/1ZJl0pSP84nLUtb31X6/ApJD0h6e2bfwHxXc/j7OyF97pb0+Uel+wfme0rL087f\n3yJJG9L9t0l6V+Y1g/RdPV7Sl9Iy3iLp/HT/UknXSLojvT0y3a/0u9gq6UZJz8i81+r0+Dskre7X\nOdUiIkZ6A34NeAZwc2bfdcDJ6lJKAAAGu0lEQVQL0vtnA3+R3j8e+C5wKPAE4E5gYbrdCfwCsDg9\n5vghOq+nA8em938Z+GHmNd8CTgYEfA546TCcU+b5vwc+Cbw9fTxQ31Wb39MhwI3A09LH48DCQfue\n5nBevwNckd4fA+4CJgfwuzoGeEZ6/wjg++k14RLggnT/BcBfpfdPT78LAc8GvpnuXwr8IL09Mr1/\nZD+/r25uI1/TiIivAruadj8F+Gp6/xrglen9M0n+uB+OiH8HtgInpdvWiPhBROwBrkiP7Zt2zisi\nboiIH6X7bwEeJelQSccASyLi65H8tX8MOKv+0udr87tC0lkk/yFvyRw/UN9Vm+f0YuDGiPhu+tqd\nEbFv0L6ntGztnFcAj5Z0CHAYsAfYzeB9V/dGxLfT+/cDtwGPS8u0IT1sA7P/9mcCH4vEN4DHpt/V\nS4BrImJXRPyE5N/itB6eSq1GPmgUuBk4I73/W8Dj0/uPA+7JHLct3Ve0f9AUnVfWK4EbIuJhknPY\nlnluEM8r95wkPRp4J3Bh0/HD8F0VfU9PBkLS1ZK+Lem/p/uH4XuC4vP6FPAgcC9wN/DXEbGLAf6u\nJE2S1NC/CRwdEfdCEliAo9LDhv16MSfzNWicDfy+pOtJqqF70v157cRRsn/QFJ0XAJKeCvwV8HuN\nXTnvMWjnVXROFwL/MyIeaDp+mM/pEOB5wHR6+3JJL2I4zgmKz+skYB9wLEmz79sk/QIDel6SDidp\n9nxLROwuOzRn3zBdL+ZkXq7cFxHfI2kKQNKTgd9In9rGgb/OjwMazTpF+wdGyXkh6TjgH4DXRcSd\n6e5tJOfSMHDnVXJOzwJeJekS4LHAfkn/BVzPgH9XLf7+vhIRO9LnriLJG2xiwL8nKD2v3wE+HxF7\ngfskfQ2YIvk1PlDflaRFJAFjc0R8Ot39Y0nHRMS9afPTfen+ouvFNuC/Ne3/cp3l7qV5WdOQdFR6\nuwD4I+Cy9Kkrgdek7f1PAJ5EkoC8DniSpCdIWgy8Jj12oBSdl6THAp8F3hURX2scn1a175f07LQ3\nzuuAz/S84CWKzikinh8RkxExCbwPeHdEvJ8h+K5K/v6uBk6QNJa2/78AuHUYvicoPa+7gRemvY0e\nTZI0/h4D9l2l/7YfAW6LiPdmnroSaPSAWs3sv/2VwOvS83o28LP0u7oaeLGkI9OeVi9O942Gfmfi\n696AvyNpS91L8gvgjcD5JD0jvg9cTDrIMT1+LUmPjtvJ9FAh6Snx/fS5tcN0XiT/gR8EvpPZjkqf\nmyJpi74TeH/232KQz6npdX9G2ntq0L6rOfz9rSRJ7N8MXJLZPzDf0xz+/g4n6eF2C3Ar8I4B/a6e\nR9KMdGPm/8npJL3YrgXuSG+XpscL+EBa9puAqcx7nU3SkWYr8IZ+nle3N48INzOzyuZl85SZmc2N\ng4aZmVXmoGFmZpU5aJiZWWUOGmZmVpmDhs1raR/7f5P00sy+V0v6fBc/408kfSfd9mXu/37Ja66Q\n9JvdKoNZt7jLrc17kn6ZZBzB00lmXv0OcFrMjpyfy3seEhGPNO8DdkTEYyu8/gpgU0T881zLYFYH\n1zRs3ouIm4F/IpkA8U9JZi69M10T4VtpreCD6UhnJK2XtCVdc+FPGu8jaZukP06nyXh5lc+W9ERJ\nX07XY/iCpGNzjnmPpA81Rh5L+qqSdVGuyozC/oakiyVdp2R9imd14Z/G7CAOGmaJC0nmSHopcEla\n+3g58JyI+FWSedpekx57QURMAU8DTpV0fOZ9HoyI50bEJyt+7mXA/4qIE4B/BLLTV0jSpSRrTZxD\nss7Le4GzIuJE4OMko+Eb9kXEM4F3AX9c8fPN2jIvJyw0axYRD0r6OPBARDws6RTgmcCWZEoiDmN2\nuuvXSnojyf+fY0kW6rk1fe7jbX70FOkkfyRrNfxR5rm/BL4aEX8AP29GOx74l7RMC0kWNGpoTLB3\nPfDnbZbDrBIHDbNZ+9MNknmFLo+IA36xS3oSyRxLJ0XETyVtAh6VOeTBqh+WTpDXLJtk/CbwLEmP\niYifpWW6ISJ+veAtH05v9+H/21YTN0+Z5fsi8GpJywAkjUtaASwB7gd2a3aVtjmJpBfKdcCr0l2r\nmF35DpJZVC8F/jmdHfZmYFLSiWmZFjc1jZnVzr9GzHJExE2SLgS+mCbA95LkFbaQNEXdTLLU7NeK\n36WSc4HLJf0x8B/A65vKsUnSESRrobyMZEW8S5UsFLQQeA+zTWNmtXOXWzMzq8zNU2ZmVpmDhpmZ\nVeagYWZmlTlomJlZZQ4aZmZWmYOGmZlV5qBhZmaVOWiYmVll/x9t47vzdWhU6wAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbcd75929b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(range(1900,2015), [pc.pmi(str(i)) for i in range(1900, 2015)], 'ro')\n",
    "#plt.axis([0, 6, 0, 20])\n",
    "plt.xlabel('Year Token')\n",
    "plt.ylabel('PMI with Women Articles')\n",
    "#plt.title('Histogram of IQ')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_words_with_pmi.sort(key = lambda x: x[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('nbay', 26272, -7.611021034611001),\n",
       " ('soccerbase', 48215, -6.714110080527057),\n",
       " ('touchdowns', 21743, -5.717061075755479),\n",
       " ('cfb', 41858, -5.591893589120769),\n",
       " ('cfl', 27482, -4.883459824010785),\n",
       " ('ahl', 48558, -4.728768699616124),\n",
       " ('basho', 27121, -4.485825215739609),\n",
       " ('phillies', 18151, -4.408029612896608),\n",
       " ('ffffbf', 22239, -4.308870589754527),\n",
       " ('touchdown', 20951, -4.146025227102477),\n",
       " ('nhl', 152440, -3.980216647530143),\n",
       " ('packers', 17685, -3.8653300367238232),\n",
       " ('mlb', 112927, -3.765953985535786),\n",
       " ('nrl', 17001, -3.7100535643968486),\n",
       " ('ffdf9f', 30040, -3.667509684331198),\n",
       " ('dfdfdf', 30974, -3.6090002204332388),\n",
       " ('redskins', 17049, -3.541022692597691),\n",
       " ('sumo', 17618, -3.47376878974522),\n",
       " ('nfl', 198100, -3.362192971339228),\n",
       " ('braves', 21585, -3.3201713681263234),\n",
       " ('ligue', 20050, -3.3153947643584134),\n",
       " ('atp', 23513, -3.2587440084557056),\n",
       " ('calcio', 36112, -3.251651673848447),\n",
       " ('cardinals', 41157, -3.2000989189914497),\n",
       " ('hurling', 42983, -3.1796980729197557)]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_words_with_pmi[:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('girl', 81774, 1.3043311692138673),\n",
       " ('karen', 18868, 1.3107377579413537),\n",
       " ('rachel', 19619, 1.320925870895616),\n",
       " ('jennifer', 24554, 1.3246692332215506),\n",
       " ('amy', 21268, 1.3468364575604284),\n",
       " ('laura', 28393, 1.3497051699823768),\n",
       " ('kate', 25335, 1.361659806311657),\n",
       " ('julie', 19482, 1.3677288569117696),\n",
       " ('lisa', 24538, 1.3794512862249828),\n",
       " ('gender', 20206, 1.3821934743321176),\n",
       " ('woman', 117209, 1.4085554659874795),\n",
       " ('gymnastics', 27011, 1.4361599930835922),\n",
       " ('girls', 57627, 1.443666437110494),\n",
       " ('jessica', 17538, 1.4518054969917902),\n",
       " ('michelle', 17688, 1.4548538222469443),\n",
       " ('miss', 83631, 1.4843674303589847),\n",
       " ('husband', 75006, 1.509566866723068),\n",
       " ('ladies', 25061, 1.5567496105259568),\n",
       " ('actress', 147860, 1.7006927238502563),\n",
       " ('her', 1308263, 1.7244667675788659),\n",
       " ('women', 532471, 1.742348800649344),\n",
       " ('female', 123970, 1.7434816438590204),\n",
       " ('herself', 41532, 1.75370262315033),\n",
       " ('she', 1412916, 1.7940272570088585),\n",
       " ('actresses', 93472, 1.8524058451615915)]"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_words_with_pmi[-25:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.057060278138187925"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"wife\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.509566866723068"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"husband\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.3782836948463633"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"calculus\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.43455580425201745"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"algebra\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.7521868352706473"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"geometry\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.9148179789333222"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"manifold\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.8668498910403404"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"vector\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.438609525439805"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"killed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.24261275656700754"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"python\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.3288831679724957"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"java\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.0698052654783705"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"c\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.5527119286370876"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"javascript\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.pmi(\"adascript\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.07980790465355447, -0.4304942959128377, 1.1057827364354074]"
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pc.pmi(word) for word in ['racist', 'bigot', 'sexist']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Category Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "872ca97032ce49aabb371f06ce5e6b12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39ce5d9b28744239b323a8ea1457d25c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fp = FloatProgress(min=0, max=850399)\n",
    "cc = IntText(value=0, description=\"Categories\")\n",
    "display(cc, fp)\n",
    "article_count = 0\n",
    "category_counts = None\n",
    "\n",
    "matcher = re.compile(\"\\[\\[Category:([^\\]]*)\", re.IGNORECASE)\n",
    "\n",
    "category_to_articles = collections.defaultdict(list)\n",
    "\n",
    "with open(\"data/gendered-labeled-articles\", 'r') as f:\n",
    "    for line in f:\n",
    "        article_count += 1\n",
    "        if article_count % 1000 == 0:\n",
    "            fp.value = article_count\n",
    "            cc.value = len(category_to_articles)\n",
    "        article_id = int(line.split(\" \", 1)[0])\n",
    "        for category in matcher.findall(line):\n",
    "            category_to_articles[category].append(article_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('sports', 22673),\n",
       " ('monarch', 12644),\n",
       " ('scientist', 10384),\n",
       " ('artists', 30265),\n",
       " ('programming', 1047),\n",
       " ('fictional', 4)]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_to_articles = collections.defaultdict(set)\n",
    "topic_patterns = {\n",
    "    \"programming\": re.compile(\"software|programmer\"),\n",
    "    \"sports\": re.compile(\"athlete|sportspeople\"),\n",
    "    \"artists\": re.compile(\"artist\"),\n",
    "    \"scientist\": re.compile(\"scientist\"),\n",
    "    \"fictional\": re.compile(\"fictional\"),\n",
    "    \"monarch\": re.compile(\"king|queen|monarch|royal|prince|princess\"),\n",
    "    \n",
    "    \n",
    "    #                          \"politicians\",\n",
    "     #                         \"military personnel\",\n",
    "\n",
    "                              #sportmanager\n",
    "                              #cleric\n",
    "       #                       \"monarch\",\n",
    "        #                      \"Fictional\\ characters\",\n",
    "         #                     \"nobility\",\n",
    "          #                    \"criminals\",\n",
    "           #                   \"judges\"\n",
    "    \n",
    "}\n",
    "\n",
    "\n",
    "for k,v in category_to_articles.items():\n",
    "    for topic, pattern in topic_patterns.items():\n",
    "        if pattern.search(k):\n",
    "            topic_to_articles[topic].update(v)\n",
    "            \n",
    "[(k, len(v)) for (k,v) in topic_to_articles.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[21781, 10120, 995, 8438, 17528]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len([i for i in topic_to_articles[topic] if is_male[str(i)]]) for topic in topics]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b570bd234784e8c9f030709e70271ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>FloatProgress</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "FloatProgress(value=0.0, max=850399.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Articles: 850399 Time: 7.9872894287109375 seconds\n"
     ]
    }
   ],
   "source": [
    "fp = FloatProgress(min=0, max=850399)\n",
    "display(fp)\n",
    "article_count = 0\n",
    "\n",
    "topic_writers = {topic: open(\"data/topic-\" + topic + \"words\", 'w') for topic in topic_to_articles.keys()}\n",
    "\n",
    "start = time.time()\n",
    "try:\n",
    "    with open(\"data/gendered-labeled-arlticles.stripped\", 'r') as f:\n",
    "        for line in f:\n",
    "            article_count += 1\n",
    "            if article_count % 1000 == 0:\n",
    "                fp.value = article_count\n",
    "            article_id, words = line.split(\" \", 1)\n",
    "            for topic, ids in topic_to_articles.items():\n",
    "                if article_id in ids:\n",
    "                    topic_writers[topic].write(words)\n",
    "                    topic_writers[topic].write(\"\\n\")\n",
    "\n",
    "finally:        \n",
    "    for writer in topic_writers.values():\n",
    "        writer.close()\n",
    "    print(\"Articles:\", article_count, \"Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_glove_file(file_path, ndim):\n",
    "    # File path inside archive\n",
    "    inner_path = \"glove.6B.{:d}d.txt\".format(ndim)\n",
    "    print(\"Parsing file: {:s}:{:s}\".format(file_path, inner_path))\n",
    "    # Count lines to pre-allocate memory\n",
    "    line_count = 0\n",
    "    with open(file_path, 'r') as f:\n",
    "        for line in f:\n",
    "            line_count += 1\n",
    "    print(\"Found {:,} words.\".format(line_count))\n",
    "    \n",
    "    # Pre-allocate vectors as a contiguous array\n",
    "    W = np.zeros((line_count, ndim), dtype=np.float32)\n",
    "    words = []\n",
    "\n",
    "    print(\"Parsing vectors... \", end=\"\")\n",
    "    with open(file_path, 'r') as f:\n",
    "        for i, line in enumerate(f):\n",
    "            word, numbers = line.split(maxsplit=1)\n",
    "            words.append(word)\n",
    "            W[i] = np.fromstring(numbers, dtype=np.float32, sep=\" \")\n",
    "    \n",
    "    print(\"Done! (W.shape = {:s})\".format(str(W.shape)))\n",
    "    return words, W\n",
    "\n",
    "def find_nn_cos(v, Wv, k=10):\n",
    "    cos_similarity = np.dot(Wv, v) / (np.linalg.norm(v) * np.linalg.norm(Wv, axis=1))\n",
    "    nns = np.argsort(cos_similarity)[-k:]\n",
    "    return (nns, cos_similarity[nns])\n",
    "\n",
    "def analogy(vA, vB, vC, Wv, k=5):\n",
    "    return find_nn_cos(vB - vA + vC, Wv, k)\n",
    "\n",
    "class Glove:\n",
    "    \n",
    "    def __init__(self, file_path, ndim):\n",
    "        self.words, W = parse_glove_file(file_path, ndim)\n",
    "        self.W = W / np.linalg.norm(W, axis=1)[:,None]\n",
    "        self.word_index = {}\n",
    "        for i, word in enumerate(self.words):\n",
    "            self.word_index[word] = i\n",
    "            \n",
    "    def vector(self, word):\n",
    "        return self.W[self.word_index[word]]\n",
    "    \n",
    "    def analogy(self, a, b, c, k=5):\n",
    "        return [self.words[i] for i in analogy(self.vector(a), self.vector(b), self.vector(c), self.W, k)[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing file: data/glove/topic-artistswords-vectors.txt:glove.6B.200d.txt\n",
      "Found 172,878 words.\n",
      "Parsing vectors... Done! (W.shape = (172878, 200))\n",
      "Parsing file: data/glove/topic-monarchwords-vectors.txt:glove.6B.200d.txt\n",
      "Found 101,333 words.\n",
      "Parsing vectors... Done! (W.shape = (101333, 200))\n",
      "Parsing file: data/glove/topic-programmingwords-vectors.txt:glove.6B.200d.txt\n",
      "Found 17,607 words.\n",
      "Parsing vectors... Done! (W.shape = (17607, 200))\n",
      "Parsing file: data/glove/topic-scientistwords-vectors.txt:glove.6B.200d.txt\n",
      "Found 81,276 words.\n",
      "Parsing vectors... Done! (W.shape = (81276, 200))\n",
      "Parsing file: data/glove/topic-sportswords-vectors.txt:glove.6B.200d.txt\n",
      "Found 91,223 words.\n",
      "Parsing vectors... Done! (W.shape = (91223, 200))\n"
     ]
    }
   ],
   "source": [
    "glove = {}\n",
    "topics = [\"artists\", \"monarch\", \"programming\", \"scientist\", \"sports\"]\n",
    "glove = {topic: Glove(\"data/glove/topic-\" + topic + \"words-vectors.txt\", 200) for topic in topics}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing file: data/glove.6B.200d.txt:glove.6B.200d.txt\n",
      "Found 400,001 words.\n",
      "Parsing vectors... Done! (W.shape = (400001, 200))\n"
     ]
    }
   ],
   "source": [
    "glove[\"all\"] = Glove(\"data/glove.6B.200d.txt\", 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['prince', 'elizabeth', 'princess', 'queen', 'king']"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"king\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['dorovskikh', 'motley', 'sycv', 'pidhrushna', 'jock']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"sports\"].analogy(\"he\", \"she\", \"jock\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17607, 200)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"programming\"].W.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_nn_cos_with_threshold(a, b, x, Wv, threshold = 0.6):\n",
    "    ab = a - b\n",
    "    xys = Wv - x\n",
    "    cos_similarity = np.dot(xys, ab) / (np.linalg.norm(ab) * np.linalg.norm(xys, axis=1))\n",
    "    nn = np.nanargmax(cos_similarity)\n",
    "    return (nn, cos_similarity[nn])\n",
    "\n",
    "def find_all_pairs_with_threshold(a, b, Wv, threshold):\n",
    "    matches = []\n",
    "    for i in range(Wv.shape[0]):\n",
    "    #for i in range(50):\n",
    "        if i % 10000 == 0:\n",
    "            print(i)\n",
    "        j, d = find_nn_cos_with_threshold(a, b, Wv[i], Wv, threshold)\n",
    "        if d > threshold:\n",
    "            matches += [(i, j, d)]\n",
    "        \n",
    "    return matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#def find_closest_cos(ai, bi, xi, W, delta):\n",
    "#    v = W[bi] - W[ai] + W[xi]\n",
    "#    distances = np.linalg.norm(W - W[xi], axis=1)\n",
    "#    cos_similarity = np.dot(W, v) / (np.linalg.norm(v) * np.linalg.norm(W, axis=1))\n",
    "#    cos_similarity[[ai, bi, xi]] = -np.inf\n",
    "#    cos_similarity[distances > delta] = 0\n",
    "#    nn = np.nanargmax(cos_similarity)\n",
    "#    return (nn, cos_similarity[nn])\n",
    "\n",
    "def find_closest_cos(ai, bi, xi, W, delta):\n",
    "    v = W[bi] - W[ai]\n",
    "    WminusX = W - W[xi]\n",
    "    distances = np.linalg.norm(WminusX, axis=1)\n",
    "    cos_similarity = np.dot(WminusX, v) / (np.linalg.norm(v) * distances)\n",
    "    cos_similarity[[ai, bi, xi]] = -np.inf\n",
    "    cos_similarity[distances > delta] = 0\n",
    "    nn = np.nanargmax(cos_similarity)\n",
    "    return (nn, cos_similarity[nn])\n",
    "\n",
    "def find_all_pairs(glove, worda, wordb):\n",
    "    ai = glove.word_index[worda]\n",
    "    bi = glove.word_index[wordb]\n",
    "    \n",
    "    matches = []\n",
    "    start = time.time()\n",
    "    for i in range(glove.W.shape[0]):\n",
    "    #for i in range(50):\n",
    "        if i % 5000 == 0:\n",
    "            print((time.time() - start) / 60, i)\n",
    "        j, d = find_closest_cos(ai, bi, i, glove.W, 1)\n",
    "        matches += [(glove.words[i], glove.words[j], d)]\n",
    "        \n",
    "    return matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/adereth/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:15: RuntimeWarning: invalid value encountered in true_divide\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2139, 0.50673437)"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = glove[\"sports\"]\n",
    "find_closest_cos(g.word_index[\"he\"],\n",
    "                g.word_index[\"she\"],\n",
    "                g.word_index[\"brother\"],\n",
    "                 g.W,\n",
    "                 1\n",
    "                 \n",
    "                 \n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2139"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g.word_index[\"sister\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.sum(np.linalg.norm(g.W - g.W[5], axis=1) < 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.99999994,  1.        ,  1.        , ...,  1.        ,\n",
       "        1.        ,  1.        ], dtype=float32)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(\n",
    "    glove[\"scientist\"].W / np.linalg.norm(glove[\"scientist\"].W, \n",
    "                                          axis=1\n",
    "                                         )[:,None],\n",
    "    axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "programming\n",
      "2.0265579223632812e-07 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/adereth/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:16: RuntimeWarning: invalid value encountered in true_divide\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8451705892880758 5000\n",
      "1.6924994985262554 10000\n",
      "2.5346781969070435 15000\n"
     ]
    }
   ],
   "source": [
    "all_pairs = {}\n",
    "for topic in [\"programming\"]:\n",
    "    print(topic)\n",
    "    all_pairs[topic] = find_all_pairs(glove[topic], \"he\", \"she\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.2715657552083333e-07 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/adereth/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:16: RuntimeWarning: invalid value encountered in true_divide\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.725476837158203 5000\n",
      "17.45020626783371 10000\n",
      "26.215900897979736 15000\n",
      "34.93063323497772 20000\n",
      "43.66434663931529 25000\n",
      "52.37640450398127 30000\n",
      "61.141208843390146 35000\n",
      "69.96290733416875 40000\n",
      "78.7992027759552 45000\n",
      "87.59596070845922 50000\n",
      "96.40190204779307 55000\n",
      "105.4623383919398 60000\n",
      "115.16285643577575 65000\n",
      "124.87523929278056 70000\n",
      "134.66268447240193 75000\n",
      "144.389757390817 80000\n"
     ]
    }
   ],
   "source": [
    "topic = \"scientist\"\n",
    "all_pairs[topic] = find_all_pairs(glove[topic], \"he\", \"she\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.7418136596679685e-07 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/adereth/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:16: RuntimeWarning: invalid value encountered in true_divide\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.15351984500885 5000\n"
     ]
    }
   ],
   "source": [
    "topic = \"sports\"\n",
    "all_pairs[topic] = find_all_pairs(glove[topic], \"he\", \"she\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "topic = \"monarch\"\n",
    "all_pairs[topic] = find_all_pairs(glove[topic], \"he\", \"she\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "monarch\n",
      "1.3907750447591147e-07 0\n",
      "6.082759253184 5000\n",
      "12.221903717517852 10000\n",
      "18.199185117085776 15000\n",
      "24.055060120423637 20000\n",
      "29.968611180782318 25000\n",
      "35.847600956757866 30000\n",
      "41.70102709929149 35000\n",
      "47.523171166578926 40000\n",
      "53.364270170529686 45000\n",
      "59.38480354944865 50000\n",
      "65.3795783996582 55000\n",
      "71.34915070931116 60000\n",
      "77.35413228273391 65000\n",
      "83.34696946144103 70000\n",
      "89.3110499382019 75000\n",
      "95.27120043039322 80000\n",
      "101.1996314883232 85000\n",
      "107.13214097420375 90000\n",
      "113.09260906775792 95000\n",
      "119.10006855328878 100000\n",
      "programming\n",
      "1.0331471761067709e-07 0\n",
      "0.5542784293492635 5000\n",
      "1.1040859301884969 10000\n",
      "1.6476761857668558 15000\n",
      "scientist\n",
      "1.1126200358072917e-07 0\n",
      "4.8474891662597654 5000\n",
      "9.713284941514333 10000\n",
      "14.58920597632726 15000\n",
      "19.451586826642355 20000\n",
      "24.306026800473532 25000\n",
      "29.177290447553 30000\n",
      "34.04618299007416 35000\n",
      "38.91820683081945 40000\n",
      "43.845380612214406 45000\n",
      "48.756148028373715 50000\n",
      "53.722789227962494 55000\n",
      "58.61424360275269 60000\n",
      "63.53887449900309 65000\n",
      "68.47770372231801 70000\n",
      "73.42332124312719 75000\n",
      "78.36133846044541 80000\n",
      "sports\n",
      "1.1126200358072917e-07 0\n",
      "5.506797043482463 5000\n",
      "11.018625171979268 10000\n",
      "16.608082282543183 15000\n",
      "22.173291726907095 20000\n",
      "27.685607103506722 25000\n",
      "33.240590544541675 30000\n",
      "38.769757580757144 35000\n",
      "44.31979482968648 40000\n",
      "49.86390777428945 45000\n",
      "55.37881539662679 50000\n",
      "60.89212180773417 55000\n",
      "66.40572461287181 60000\n",
      "71.91859598954518 65000\n",
      "77.42950276136398 70000\n",
      "82.99095415671667 75000\n",
      "88.52754263480504 80000\n",
      "94.03354343175889 85000\n",
      "99.50364073912303 90000\n"
     ]
    }
   ],
   "source": [
    "for topic in [\"monarch\", \"programming\", \"scientist\", \"sports\"]:\n",
    "    print(topic)\n",
    "    all_pairs[topic] = find_all_pairs(glove[topic], \"he\", \"she\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('his', 'her', 0.83427042),\n",
       " ('he', 'became', 0.59106821),\n",
       " ('art1', 'fothforlano2011', 0.57391375),\n",
       " ('sustprofile', 'fothforlano2011', 0.57376134),\n",
       " ('himself', 'herself', 0.57215446),\n",
       " ('psb542', 'nationalreviewkathryn', 0.57116807),\n",
       " ('which', 'her', 0.55048639),\n",
       " ('cmu01', 'fothforlano2011', 0.55034173),\n",
       " ('aasmi', 'nationalreviewkathryn', 0.54930079),\n",
       " ('psb548', 'nationalreviewkathryn', 0.54807431),\n",
       " ('wral', 'harkewicz', 0.54374635),\n",
       " ('to', 'her', 0.53976548),\n",
       " ('trahair89', 'roanokecollegepress', 0.53280824),\n",
       " ('bezahra', 'nationalreviewkathryn', 0.5317629),\n",
       " ('dmnp', 'kexpinterview', 0.53169763),\n",
       " ('obita', 'nationalreviewkathryn', 0.52946758),\n",
       " ('haaretz1', 'fothforlano2011', 0.52799684),\n",
       " ('psb543', 'fothforlano2011', 0.52793193),\n",
       " ('fld', 'fothforlano2011', 0.52683753),\n",
       " ('him', 'her', 0.52608728),\n",
       " ('aaho', 'nationalreviewkathryn', 0.5241307),\n",
       " ('karl512', 'kexpinterview', 0.52412248),\n",
       " ('odnb1', 'fothforlano2011', 0.52391869),\n",
       " ('this', 'her', 0.52366453),\n",
       " ('bloomberg2', 'nationalreviewkathryn', 0.52199572),\n",
       " ('ldn', 'fothforlano2011', 0.51870215),\n",
       " ('emlen1981', 'fothforlano2011', 0.51783991),\n",
       " ('wollearn', 'fothforlano2011', 0.51779979),\n",
       " ('wrs', 'nationalreviewkathryn', 0.51680279),\n",
       " ('but', 'her', 0.5144999),\n",
       " ('avramov1', 'fothforlano2011', 0.51314735),\n",
       " ('europlanet', 'roanokecollegepress', 0.51285106),\n",
       " ('gilbert02', 'fothforlano2011', 0.51240575),\n",
       " ('newgenbio', 'fothforlano2011', 0.51035225),\n",
       " ('dinghy', 'seamstress', 0.51000655),\n",
       " ('nydn', 'nationalreviewkathryn', 0.50664127),\n",
       " ('psb544', 'nationalreviewkathryn', 0.5040217),\n",
       " ('pomonacollege2011commencement', 'kexpinterview', 0.50391215),\n",
       " ('some', 'her', 0.50357342),\n",
       " ('longre', 'fothforlano2011', 0.50092924),\n",
       " ('nanoint', 'fothforlano2011', 0.50065398),\n",
       " ('international320', 'foxoral', 0.50039172),\n",
       " ('odnb2', 'nationalreviewkathryn', 0.50023413),\n",
       " ('had', 'her', 0.49981692),\n",
       " ('psb547', 'bose152', 0.49754989),\n",
       " ('vaz3', 'fothforlano2011', 0.49603245),\n",
       " ('arthurpapers', 'kexpinterview', 0.49539745),\n",
       " ('ivxxxiii', 'warebraukman234', 0.49509761),\n",
       " ('sneli', 'kexpinterview', 0.4940291),\n",
       " ('labsky', 'fothforlano2011', 0.49218073),\n",
       " ('bubbling', 'scoles161004', 0.49111676),\n",
       " ('news93', 'benedictsto', 0.49008718),\n",
       " ('while', 'her', 0.48981121),\n",
       " ('was', 'her', 0.48975059),\n",
       " ('many', 'her', 0.48841226),\n",
       " ('slob', 'nationalreviewkathryn', 0.48695776),\n",
       " ('sullivanreview', 'bose152', 0.48662326),\n",
       " ('jezdinský', 'benedictsto', 0.48633522),\n",
       " ('when', 'her', 0.48593813),\n",
       " ('Şimonca', 'bdws', 0.4850125),\n",
       " ('time', 'her', 0.48412257),\n",
       " ('also', 'her', 0.48356628),\n",
       " ('completedictionary', 'fothforlano2011', 0.48289591),\n",
       " ('ntb', 'foxoral', 0.48245549),\n",
       " ('tap314', 'benedictsto', 0.4807905),\n",
       " ('work', 'her', 0.48072907),\n",
       " ('tradevibes', 'fothforlano2011', 0.4799073),\n",
       " ('ivli', 'fothforlano2011', 0.47765931),\n",
       " ('both', 'her', 0.47764069),\n",
       " ('during', 'her', 0.47653869),\n",
       " ('tuarchiv', 'nationalreviewkathryn', 0.47609699),\n",
       " ('ivxxxviii', 'fothforlano2011', 0.47546309),\n",
       " ('wpobit', 'nationalreviewkathryn', 0.47517213),\n",
       " ('tendered', 'fufa', 0.47494513),\n",
       " ('years', 'her', 0.47482917),\n",
       " ('amirikabir', 'umanch', 0.47391155),\n",
       " ('one', 'her', 0.47360769),\n",
       " ('nowak182', 'nationalreviewkathryn', 0.47296906),\n",
       " ('barsoumian', 'harkewicz', 0.47223014),\n",
       " ('were', 'her', 0.47218835),\n",
       " ('lgconvorbire', 'fothforlano2011', 0.47066221),\n",
       " ('made', 'her', 0.47061163),\n",
       " ('baignet', 'foxoral', 0.47021511),\n",
       " ('pelo', 'benedictsto', 0.47000909),\n",
       " ('these', 'her', 0.46993998),\n",
       " ('two', 'her', 0.4692657),\n",
       " ('hageman', 'kexpinterview', 0.46868813),\n",
       " ('ivliii', 'benedictsto', 0.46725532),\n",
       " ('scobit', 'bose152', 0.4668768),\n",
       " ('after', 'her', 0.4668704),\n",
       " ('reagan63', 'fothforlano2011', 0.46650276),\n",
       " ('bdi', 'encyworldsci', 0.46572426),\n",
       " ('published', 'her', 0.46519414),\n",
       " ('described', 'her', 0.46517596),\n",
       " ('psb549', 'wapost', 0.46502724),\n",
       " ('galeyev', 'roanokecollegepress', 0.46412867),\n",
       " ('however', 'her', 0.46395233),\n",
       " ('jrayo', 'fothforlano2011', 0.46389678),\n",
       " ('buikstra', 'roanokecollegepress', 0.46380278),\n",
       " ('moscovitch2001', 'bose152', 0.46330011),\n",
       " ('stlj', 'roanokecollegepress', 0.46277624),\n",
       " ('eb11', 'nationalreviewkathryn', 0.46092656),\n",
       " ('aenz', 'roanokecollegepress', 0.46085155),\n",
       " ('wilks1995', 'nationalreviewkathryn', 0.46042368),\n",
       " ('associationforlogicprogrammingobit', 'nationalreviewkathryn', 0.45858705),\n",
       " ('kravetz', 'nationalreviewkathryn', 0.45826495),\n",
       " ('heredia', 'asbury', 0.45818639),\n",
       " ('their', 'her', 0.45791867),\n",
       " ('weisenberg', 'chr18', 0.45753974),\n",
       " ('not', 'her', 0.45694879),\n",
       " ('sympathised', 'alfalfa', 0.45691556),\n",
       " ('later', 'her', 0.45679885),\n",
       " ('vrinitiati', 'nationalreviewkathryn', 0.45675775),\n",
       " ('patterson1987', 'nationalreviewkathryn', 0.45621616),\n",
       " ('aguirre125', 'fothforlano2011', 0.45604369),\n",
       " ('own', 'her', 0.45597142),\n",
       " ('lourdusamy2004', 'fothforlano2011', 0.45493597),\n",
       " ('cebio', 'fothforlano2011', 0.45473668),\n",
       " ('birkeland', 'fothforlano2011', 0.45439899),\n",
       " ('tjfaps', 'journalen', 0.45338112),\n",
       " ('at', 'her', 0.4533774),\n",
       " ('mpbanda', 'fothforlano2011', 0.45337597),\n",
       " ('studies', 'her', 0.45325193),\n",
       " ('father', 'her', 0.45171547),\n",
       " ('wwl', 'wapost', 0.45146474),\n",
       " ('eljera', 'fothforlano2011', 0.45101687),\n",
       " ('several', 'her', 0.4509083),\n",
       " ('pantheistic', 'spinster', 0.45079994),\n",
       " ('shunned', 'raped', 0.44961262),\n",
       " ('pedasfamscienceatsea', 'nationalreviewkathryn', 0.44904438),\n",
       " ('wrote', 'her', 0.44841838),\n",
       " ('hstu', 'fothforlano2011', 0.44837546),\n",
       " ('bizbdnews', 'fothforlano2011', 0.44794297),\n",
       " ('győr', 'visakhapatnam', 0.44792101),\n",
       " ('ivxxxiv', 'fothforlano2011', 0.44702777),\n",
       " ('patterson1997', 'fothforlano2011', 0.44676024),\n",
       " ('came', 'her', 0.44641116),\n",
       " ('sundari', 'fothforlano2011', 0.44481713),\n",
       " ('obscuring', 'forte', 0.44418541),\n",
       " ('jvhri', 'fothforlano2011', 0.44407561),\n",
       " ('there', 'her', 0.44401762),\n",
       " ('gajdusek', 'kankhal', 0.44373187),\n",
       " ('following', 'her', 0.44341621),\n",
       " ('same', 'her', 0.44308195),\n",
       " ('latimes20070412', 'nationalreviewkathryn', 0.44302994),\n",
       " ('pomonadotedu', 'fothforlano2011', 0.4430081),\n",
       " ('ivxxxv', 'fothforlano2011', 0.44197187),\n",
       " ('greengiant', 'nationalreviewkathryn', 0.44178513),\n",
       " ('took', 'her', 0.44173804),\n",
       " ('have', 'her', 0.44170767),\n",
       " ('continued', 'her', 0.44113919),\n",
       " ('fkameny', 'benedictsto', 0.44100136),\n",
       " ('broad20', 'fothforlano2011', 0.44084853),\n",
       " ('sisgov', 'roanokecollegepress', 0.44075745),\n",
       " ('mckie1', 'nationalreviewkathryn', 0.43982461),\n",
       " ('did', 'her', 0.43928361),\n",
       " ('diesnyt', 'fothforlano2011', 0.43926695),\n",
       " ('rosci1', 'chr18', 0.43895924),\n",
       " ('interest', 'her', 0.4388656),\n",
       " ('ap070417', 'benedictsto', 0.43827632),\n",
       " ('midgette1991', 'nationalreviewkathryn', 0.43763986),\n",
       " ('nytob', 'nationalreviewkathryn', 0.43651602),\n",
       " ('included', 'her', 0.43646023),\n",
       " ('curecord', 'fothforlano2011', 0.43635908),\n",
       " ('zagora', 'personaljourney', 0.43572104),\n",
       " ('rolle1991', 'fothforlano2011', 0.43542397),\n",
       " ('fl18', 'fothforlano2011', 0.43541285),\n",
       " ('reagan62', 'benedictsto', 0.43504322),\n",
       " ('encyclo', 'oralhistorychf', 0.43498993),\n",
       " ('didiscs', 'ozment', 0.43470311),\n",
       " ('lloyd2007', 'brockmaninterview', 0.43454832),\n",
       " ('dehaenecv', 'roanokecollegepress', 0.43361834),\n",
       " ('study', 'her', 0.43333361),\n",
       " ('tesla1', 'fothforlano2011', 0.43248206),\n",
       " ('thryptomene', 'fothforlano2011', 0.43206489),\n",
       " ('been', 'her', 0.43140739),\n",
       " ('steindór', 'nationalreviewkathryn', 0.43126017),\n",
       " ('sabu', 'roanokecollegepress', 0.43112829),\n",
       " ('nytimes1993', 'benedictsto', 0.43108353),\n",
       " ('kucera', 'womeninchemistrytranscript', 0.43095562),\n",
       " ('fnkaa', 'fothforlano2011', 0.43065241),\n",
       " ('google3', 'chr18', 0.43013766),\n",
       " ('woc', 'wapost', 0.43006971),\n",
       " ('appelrouthedles2007', 'roanokecollegepress', 0.42873216),\n",
       " ('being', 'her', 0.42872548),\n",
       " ('addition', 'her', 0.42827716),\n",
       " ('wife', 'husband', 0.42750484),\n",
       " ('cerc1', 'fothforlano2011', 0.42701948),\n",
       " ('ausubelorg', 'nationalreviewkathryn', 0.4267379),\n",
       " ('psb546', 'nationalreviewkathryn', 0.42657208),\n",
       " ('early', 'her', 0.42640448),\n",
       " ('möhring', 'fothforlano2011', 0.42616427),\n",
       " ('isihc', 'fothforlano2011', 0.42588136),\n",
       " ('lmj6', 'roanokecollegepress', 0.42584506),\n",
       " ('siddiqi2000', 'runciman20130606', 0.42459348),\n",
       " ('psb541', 'fothforlano2011', 0.42405164),\n",
       " ('inmem', 'fothforlano2011', 0.42390206),\n",
       " ('kudyba', 'nationalreviewkathryn', 0.42367676),\n",
       " ('gill2007', 'bdws', 0.42325732),\n",
       " ('began', 'her', 0.42317063)]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "sorted(all_pairs[\"scientist\"], key=lambda x: -x[-1])[0:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['grandmother', 'grandparents', 'maternal', 'great-grandmother', 'paternal']"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"paternal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for topic in [\"monarch\", \"programming\", \"scientist\", \"sports\"]:\n",
    "    w = open(\"data/topic-\" + topic + \"-he-she-analogies\", 'w')\n",
    "    for w1, w2, score in sorted(all_pairs[topic], key=lambda x: -x[-1]):\n",
    "        w.write(w1 + \" \" + w2 + \" \" + str(score) + \"\\n\" )\n",
    "    w.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/madereth/anaconda3/envs/w266-env/lib/python3.6/site-packages/ipykernel_launcher.py:4: RuntimeWarning: invalid value encountered in true_divide\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "10000\n",
      "10100\n",
      "10200\n",
      "10300\n",
      "10400\n",
      "10500\n",
      "10600\n",
      "10700\n",
      "10800\n",
      "10900\n",
      "11000\n",
      "11100\n",
      "11200\n",
      "11300\n",
      "11400\n",
      "11500\n",
      "11600\n",
      "11700\n",
      "11800\n",
      "11900\n",
      "12000\n",
      "12100\n",
      "12200\n",
      "12300\n",
      "12400\n",
      "12500\n",
      "12600\n",
      "12700\n",
      "12800\n",
      "12900\n",
      "13000\n",
      "13100\n",
      "13200\n",
      "13300\n",
      "13400\n",
      "13500\n",
      "13600\n",
      "13700\n",
      "13800\n",
      "13900\n",
      "14000\n",
      "14100\n",
      "14200\n",
      "14300\n",
      "14400\n",
      "14500\n",
      "14600\n",
      "14700\n",
      "14800\n",
      "14900\n",
      "15000\n",
      "15100\n",
      "15200\n",
      "15300\n",
      "15400\n",
      "15500\n",
      "15600\n",
      "15700\n",
      "15800\n",
      "15900\n",
      "16000\n",
      "16100\n",
      "16200\n",
      "16300\n",
      "16400\n",
      "16500\n",
      "16600\n",
      "16700\n",
      "16800\n",
      "16900\n",
      "17000\n",
      "17100\n",
      "17200\n",
      "17300\n",
      "17400\n",
      "17500\n",
      "17600\n"
     ]
    }
   ],
   "source": [
    "g = glove[\"programming\"]\n",
    "m = find_all_pairs_with_threshold(g.vector(\"she\"),\n",
    "                                  g.vector(\"he\"),\n",
    "                                  g.W,\n",
    "                                  0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(17, 235, 1.0000001),\n",
       " (27, 262, 0.6511974),\n",
       " (69, 235, 0.5440264),\n",
       " (194, 235, 0.6359928),\n",
       " (11554, 10886, 0.51645917)]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('he', 'she', 1.0000001),\n",
       " ('his', 'her', 0.6511974),\n",
       " ('also', 'she', 0.5440264),\n",
       " ('where', 'she', 0.6359928),\n",
       " ('usflinfo', 'businessinsiderbio', 0.51645917)]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(g.words[i[0]],g.words[i[1]], i[2]) for i in m]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/madereth/anaconda3/envs/w266-env/lib/python3.6/site-packages/ipykernel_launcher.py:4: RuntimeWarning: invalid value encountered in true_divide\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "28000\n",
      "29000\n",
      "30000\n",
      "31000\n",
      "32000\n",
      "33000\n",
      "34000\n",
      "35000\n",
      "36000\n",
      "37000\n",
      "38000\n",
      "39000\n",
      "40000\n",
      "41000\n",
      "42000\n",
      "43000\n",
      "44000\n",
      "45000\n",
      "46000\n",
      "47000\n",
      "48000\n",
      "49000\n",
      "50000\n",
      "51000\n",
      "52000\n",
      "53000\n",
      "54000\n",
      "55000\n",
      "56000\n",
      "57000\n",
      "58000\n",
      "59000\n",
      "60000\n",
      "61000\n",
      "62000\n",
      "63000\n",
      "64000\n",
      "65000\n",
      "66000\n",
      "67000\n",
      "68000\n",
      "69000\n",
      "70000\n",
      "71000\n",
      "72000\n",
      "73000\n",
      "74000\n",
      "75000\n",
      "76000\n",
      "77000\n",
      "78000\n",
      "79000\n",
      "80000\n",
      "81000\n"
     ]
    }
   ],
   "source": [
    "g = glove[\"scientist\"]\n",
    "ms = find_all_pairs_with_threshold(g.vector(\"she\"),\n",
    "                                  g.vector(\"he\"),\n",
    "                                  g.W,\n",
    "                                  0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 'she', 0.5233252),\n",
       " ('of', 'she', 0.5472787),\n",
       " ('in', 'she', 0.5727447),\n",
       " ('to', 'her', 0.5099384),\n",
       " ('he', 'she', 1.0),\n",
       " ('was', 'she', 0.51960063),\n",
       " ('for', 'she', 0.54781663),\n",
       " ('his', 'her', 0.8312228),\n",
       " ('which', 'she', 0.5803153),\n",
       " ('also', 'she', 0.6061445),\n",
       " ('this', 'her', 0.5041148),\n",
       " ('had', 'she', 0.5384969),\n",
       " ('one', 'she', 0.5130893),\n",
       " ('were', 'her', 0.52489454),\n",
       " ('may', 'she', 0.50021696),\n",
       " ('years', 'she', 0.52327156),\n",
       " ('been', 'she', 0.51635003),\n",
       " ('but', 'her', 0.5574061),\n",
       " ('published', 'her', 0.5312385),\n",
       " ('where', 'she', 0.6118564),\n",
       " ('became', 'she', 0.5249684),\n",
       " ('during', 'she', 0.53547204),\n",
       " ('later', 'she', 0.568919),\n",
       " ('him', 'her', 0.5112731),\n",
       " ('when', 'she', 0.5515992),\n",
       " ('then', 'she', 0.5205146),\n",
       " ('many', 'her', 0.5007815),\n",
       " ('there', 'she', 0.52698076),\n",
       " ('some', 'her', 0.5207492),\n",
       " ('worked', 'she', 0.5267218),\n",
       " ('made', 'her', 0.5040406),\n",
       " ('these', 'her', 0.5011051),\n",
       " ('while', 'she', 0.51094985),\n",
       " ('since', 'she', 0.53882056),\n",
       " ('until', 'she', 0.50220066),\n",
       " ('served', 'she', 0.5474284),\n",
       " ('several', 'she', 0.5128531),\n",
       " ('both', 'her', 0.5040101),\n",
       " ('developed', 'she', 0.508755),\n",
       " ('joined', 'she', 0.50012165),\n",
       " ('himself', 'herself', 0.5572429),\n",
       " ('came', 'her', 0.502092),\n",
       " ('jifeng', 'wtec', 0.52548295),\n",
       " ('relinquished', 'fothforlano2011', 0.5702163),\n",
       " ('psb549', 'fothforlano2011', 0.51440054),\n",
       " ('adhered', 'recited', 0.50013304),\n",
       " ('comedies', 'recited', 0.5021876),\n",
       " ('bubbling', 'wtec', 0.5302785),\n",
       " ('heinkel', 'persevered', 0.5065342),\n",
       " ('baronetcy', 'persevered', 0.5031335),\n",
       " ('baignet', 'fothforlano2011', 0.5658149),\n",
       " ('mantua', 'persevered', 0.51178986),\n",
       " ('psb547', 'fothforlano2011', 0.56944746),\n",
       " ('europlanet', 'wtec', 0.51437986),\n",
       " ('purposefully', 'journalen', 0.5026232),\n",
       " ('sneli', 'kexpinterview', 0.50219923),\n",
       " ('avramov1', 'fothforlano2011', 0.50773406),\n",
       " ('dmnp', 'kexpinterview', 0.50371075),\n",
       " ('familiarized', 'persevered', 0.5105505),\n",
       " ('fld', 'fothforlano2011', 0.5059941),\n",
       " ('ivxxxix', 'benedictsto', 0.50758857),\n",
       " ('karl512', 'kexpinterview', 0.5100727),\n",
       " ('cmu01', 'fothforlano2011', 0.55806065),\n",
       " ('psb543', 'fothforlano2011', 0.57245505),\n",
       " ('roundworms', 'wtec', 0.5084353),\n",
       " ('lycee', 'persevered', 0.5251625),\n",
       " ('psb542', 'fothforlano2011', 0.606217),\n",
       " ('psb548', 'fothforlano2011', 0.5879178),\n",
       " ('tabulated', 'recited', 0.52302545),\n",
       " ('wral', 'benedictsto', 0.5555965),\n",
       " ('bezahra', 'roanokecollegepress', 0.54864943),\n",
       " ('haaretz1', 'fothforlano2011', 0.53749377),\n",
       " ('international320', 'fothforlano2011', 0.5260462),\n",
       " ('newgenbio', 'benedictsto', 0.5405012),\n",
       " ('obita', 'roanokecollegepress', 0.5174571),\n",
       " ('odnb1', 'fothforlano2011', 0.5480099),\n",
       " ('patronized', 'recited', 0.5188191),\n",
       " ('psb544', 'nationalreviewkathryn', 0.53083944),\n",
       " ('scorpions', 'fothforlano2011', 0.5227347),\n",
       " ('sustprofile', 'fothforlano2011', 0.56675524),\n",
       " ('Şimonca', 'benedictsto', 0.5225617),\n",
       " ('aaho', 'journalen', 0.5170722),\n",
       " ('aasmi', 'fothforlano2011', 0.5184842),\n",
       " ('availed', 'persevered', 0.53828984),\n",
       " ('chided', 'persevered', 0.51200044),\n",
       " ('dinghy', 'persevered', 0.5044191),\n",
       " ('gilbert02', 'fothforlano2011', 0.5294218),\n",
       " ('ivxxxiii', 'roanokecollegepress', 0.5067938),\n",
       " ('ntb', 'fothforlano2011', 0.5403402),\n",
       " ('odnb2', 'fothforlano2011', 0.52316755),\n",
       " ('pietermaritzburg', 'persevered', 0.5264419),\n",
       " ('reprised', 'fothforlano2011', 0.503084),\n",
       " ('thoroughness', 'wtec', 0.5213962),\n",
       " ('tuxen', 'benedictsto', 0.503642),\n",
       " ('wollearn', 'fothforlano2011', 0.52885705)]"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(g.words[i[0]],g.words[i[1]], i[2]) for i in ms]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 'she', 0.579172),\n",
       " ('in', 'she', 0.55628306),\n",
       " ('ref', 'she', 0.54113567),\n",
       " ('and', 'she', 0.54077613),\n",
       " ('of', 'she', 0.53934544),\n",
       " ('a', 'she', 0.5583278),\n",
       " ('to', 'she', 0.5515262),\n",
       " ('http', 'she', 0.5076464),\n",
       " ('he', 'she', 1.0),\n",
       " ('for', 'she', 0.54486716),\n",
       " ('url', 'she', 0.5016218),\n",
       " ('cite', 'she', 0.53249764),\n",
       " ('was', 'she', 0.66509193),\n",
       " ('on', 'she', 0.56037706),\n",
       " ('his', 'her', 0.8940843),\n",
       " ('accessdate', 'she', 0.5294304),\n",
       " ('with', 'she', 0.6024914),\n",
       " ('first', 'she', 0.53099513),\n",
       " ('season', 'she', 0.5237535),\n",
       " ('league', 'she', 0.51617336),\n",
       " ('as', 'she', 0.63189346),\n",
       " ('men', 'women', 0.52905464),\n",
       " ('10', 'she', 0.5487249),\n",
       " ('5', 'she', 0.5192066),\n",
       " ('by', 'she', 0.52440053),\n",
       " ('12', 'she', 0.5518395),\n",
       " ('11', 'she', 0.56885874),\n",
       " ('club', 'she', 0.5129539),\n",
       " ('after', 'she', 0.59224343),\n",
       " ('that', 'she', 0.66932565),\n",
       " ('august', 'she', 0.5687072),\n",
       " ('an', 'she', 0.53950995),\n",
       " ('career', 'she', 0.55999166),\n",
       " ('work', 'she', 0.5178626),\n",
       " ('july', 'she', 0.55886763),\n",
       " ('8', 'she', 0.5347986),\n",
       " ('won', 'she', 0.5589703),\n",
       " ('time', 'she', 0.5126725),\n",
       " ('13', 'she', 0.5388511),\n",
       " ('20', 'she', 0.50311404),\n",
       " ('may', 'she', 0.5705042),\n",
       " ('january', 'she', 0.566734),\n",
       " ('june', 'she', 0.5494728),\n",
       " ('14', 'she', 0.53432494),\n",
       " ('15', 'she', 0.52460426),\n",
       " ('9', 'she', 0.5224163),\n",
       " ('16', 'she', 0.55426985),\n",
       " ('september', 'she', 0.58551604),\n",
       " ('march', 'she', 0.57070434),\n",
       " ('also', 'she', 0.6987141),\n",
       " ('17', 'she', 0.5393983),\n",
       " ('against', 'she', 0.5364902),\n",
       " ('one', 'she', 0.59934896),\n",
       " ('two', 'she', 0.6125996),\n",
       " ('april', 'she', 0.5677047),\n",
       " ('played', 'she', 0.6109378),\n",
       " ('october', 'she', 0.57293385),\n",
       " ('february', 'she', 0.5550094),\n",
       " ('21', 'she', 0.506869),\n",
       " ('had', 'she', 0.6477691),\n",
       " ('november', 'she', 0.5723685),\n",
       " ('18', 'she', 0.5419483),\n",
       " ('december', 'she', 0.5851945),\n",
       " ('it', 'she', 0.5135647),\n",
       " ('19', 'she', 0.5527859),\n",
       " ('second', 'she', 0.5434445),\n",
       " ('23', 'she', 0.5099889),\n",
       " ('but', 'she', 0.68821704),\n",
       " ('who', 'she', 0.56507206),\n",
       " ('22', 'she', 0.5130135),\n",
       " ('24', 'she', 0.50619435),\n",
       " ('game', 'she', 0.54787546),\n",
       " ('25', 'she', 0.5295107),\n",
       " ('30', 'she', 0.52398926),\n",
       " ('up', 'she', 0.539704),\n",
       " ('27', 'she', 0.5345747),\n",
       " ('28', 'she', 0.5341827),\n",
       " ('26', 'she', 0.52227974),\n",
       " ('29', 'she', 0.51955646),\n",
       " ('years', 'she', 0.549423),\n",
       " ('1999', 'she', 0.5013601),\n",
       " ('not', 'she', 0.5341195),\n",
       " ('during', 'she', 0.623261),\n",
       " ('three', 'she', 0.55752623),\n",
       " ('which', 'she', 0.6197448),\n",
       " ('made', 'her', 0.62518984),\n",
       " ('him', 'her', 0.67044705),\n",
       " ('has', 'she', 0.53701407),\n",
       " ('out', 'she', 0.5202438),\n",
       " ('over', 'she', 0.5324733),\n",
       " ('before', 'she', 0.5287382),\n",
       " ('where', 'she', 0.6944412),\n",
       " ('31', 'she', 0.51619786),\n",
       " ('1997', 'she', 0.5133096),\n",
       " ('be', 'she', 0.5161933),\n",
       " ('this', 'she', 0.56554306),\n",
       " ('when', 'she', 0.6614316),\n",
       " ('scored', 'she', 0.5401209),\n",
       " ('only', 'she', 0.51327515),\n",
       " ('finished', 'she', 0.552914),\n",
       " ('later', 'she', 0.69688374),\n",
       " ('end', 'she', 0.5128192),\n",
       " ('were', 'she', 0.5408808),\n",
       " ('four', 'she', 0.55653024),\n",
       " ('former', 'she', 0.5048808),\n",
       " ('play', 'she', 0.5438143),\n",
       " ('playing', 'she', 0.54047686),\n",
       " ('third', 'she', 0.514622),\n",
       " ('signed', 'she', 0.58311933),\n",
       " ('their', 'her', 0.56544673),\n",
       " ('would', 'she', 0.5839583),\n",
       " ('back', 'she', 0.52583075),\n",
       " ('then', 'she', 0.6508393),\n",
       " ('debut', 'she', 0.5456306),\n",
       " ('contract', 'she', 0.5612896),\n",
       " ('home', 'she', 0.51983213),\n",
       " ('following', 'she', 0.5994905),\n",
       " ('became', 'she', 0.5877889),\n",
       " ('they', 'she', 0.53393406),\n",
       " ('into', 'she', 0.5014889),\n",
       " ('both', 'she', 0.56407005),\n",
       " ('most', 'she', 0.52982634),\n",
       " ('have', 'she', 0.54428524),\n",
       " ('been', 'she', 0.5470756),\n",
       " ('more', 'she', 0.53722775),\n",
       " ('being', 'she', 0.6158609),\n",
       " ('while', 'her', 0.6093814),\n",
       " ('however', 'she', 0.56096584),\n",
       " ('did', 'she', 0.5112971),\n",
       " ('five', 'she', 0.55458415),\n",
       " ('off', 'her', 0.51206666),\n",
       " ('again', 'she', 0.60793245),\n",
       " ('early', 'she', 0.59329),\n",
       " ('manager', 'she', 0.5226288),\n",
       " ('part', 'she', 0.5203587),\n",
       " ('took', 'she', 0.5364619),\n",
       " ('until', 'she', 0.6108756),\n",
       " ('scoring', 'she', 0.52963895),\n",
       " ('came', 'she', 0.5407095),\n",
       " ('returned', 'she', 0.5740939),\n",
       " ('six', 'she', 0.5434029),\n",
       " ('joined', 'she', 0.5720797),\n",
       " ('went', 'she', 0.5874814),\n",
       " ('side', 'she', 0.53495836),\n",
       " ('started', 'she', 0.54365736),\n",
       " ('since', 'she', 0.56288236),\n",
       " ('head', 'she', 0.50968504),\n",
       " ('serie', 'she', 0.5128376),\n",
       " ('well', 'she', 0.6170354),\n",
       " ('began', 'she', 0.51575136),\n",
       " ('deal', 'she', 0.5145981),\n",
       " ('next', 'she', 0.52808076),\n",
       " ('footballer', 'she', 0.50074047),\n",
       " ('just', 'she', 0.53443146),\n",
       " ('return', 'her', 0.5094534),\n",
       " ('seasons', 'she', 0.54976016),\n",
       " ('moved', 'she', 0.54324865),\n",
       " ('said', 'she', 0.52893126),\n",
       " ('another', 'she', 0.5728946),\n",
       " ('led', 'she', 0.5430461),\n",
       " ('having', 'she', 0.60188854),\n",
       " ('there', 'she', 0.57683605),\n",
       " ('same', 'she', 0.55330455),\n",
       " ('despite', 'her', 0.55130446),\n",
       " ('johnson', 'she', 0.5059558),\n",
       " ('known', 'she', 0.5447069),\n",
       " ('seven', 'she', 0.50856555),\n",
       " ('days', 'she', 0.54559594),\n",
       " ('father', 'her', 0.56770396),\n",
       " ('still', 'she', 0.55226415),\n",
       " ('several', 'she', 0.5426365),\n",
       " ('brother', 'sister', 0.5468202),\n",
       " ('making', 'her', 0.56378365),\n",
       " ('released', 'her', 0.52989584),\n",
       " ('them', 'she', 0.5633349),\n",
       " ('because', 'she', 0.5281626),\n",
       " ('could', 'she', 0.54805696),\n",
       " ('called', 'she', 0.55457455),\n",
       " ('ended', 'she', 0.52385163),\n",
       " ('months', 'she', 0.5051946),\n",
       " ('starting', 'she', 0.5120178),\n",
       " ('move', 'her', 0.50342774),\n",
       " ('although', 'she', 0.5754556),\n",
       " ('late', 'she', 0.5177169),\n",
       " ('himself', 'she', 0.54321116),\n",
       " ('hit', 'she', 0.5491018),\n",
       " ('many', 'she', 0.53120613),\n",
       " ('plays', 'she', 0.5052599),\n",
       " ('needed', 'she', 0.54249156),\n",
       " ('some', 'she', 0.53190166),\n",
       " ('leading', 'she', 0.50129443),\n",
       " ('along', 'she', 0.535262),\n",
       " ('signing', 'she', 0.50639397),\n",
       " ('never', 'she', 0.5544864),\n",
       " ('giants', 'she', 0.5263767),\n",
       " ('once', 'she', 0.5666841),\n",
       " ('helped', 'she', 0.50938946),\n",
       " ('ball', 'she', 0.5157628),\n",
       " ('spent', 'she', 0.572317),\n",
       " ('campaign', 'she', 0.5025246),\n",
       " ('twice', 'she', 0.51539254),\n",
       " ('without', 'her', 0.5117553),\n",
       " ('served', 'she', 0.54131436),\n",
       " ('continued', 'she', 0.51438737),\n",
       " ('become', 'she', 0.5116476),\n",
       " ('saw', 'she', 0.5329279),\n",
       " ('going', 'she', 0.500112),\n",
       " ('agent', 'she', 0.5135391),\n",
       " ('eventually', 'she', 0.52904606),\n",
       " ('own', 'her', 0.5456018),\n",
       " ('worked', 'she', 0.5358004),\n",
       " ('even', 'she', 0.5538841),\n",
       " ('included', 'she', 0.5231765),\n",
       " ('got', 'she', 0.52179307),\n",
       " ('used', 'she', 0.51363796),\n",
       " ('each', 'she', 0.5056055),\n",
       " ('such', 'she', 0.5017099),\n",
       " ('though', 'she', 0.5830479),\n",
       " ('participated', 'she', 0.5082962),\n",
       " ('managed', 'she', 0.51684785),\n",
       " ('decided', 'she', 0.5017851),\n",
       " ('few', 'she', 0.5051269),\n",
       " ('plate', 'she', 0.5171092),\n",
       " ('prior', 'she', 0.533207),\n",
       " ('followed', 'she', 0.5476807),\n",
       " ('subsequently', 'she', 0.5369748),\n",
       " ('joining', 'she', 0.5083621),\n",
       " ('able', 'she', 0.5003742),\n",
       " ('remained', 'she', 0.56903505),\n",
       " ('sent', 'she', 0.5241089),\n",
       " ('previously', 'she', 0.5223881),\n",
       " ('leaving', 'she', 0.5050916),\n",
       " ('claimed', 'she', 0.5110634),\n",
       " ('spell', 'she', 0.52004516),\n",
       " ('offensive', 'she', 0.50221086),\n",
       " ('addition', 'she', 0.5370049),\n",
       " ('moving', 'she', 0.50678754),\n",
       " ('instead', 'she', 0.5239799),\n",
       " ('stay', 'she', 0.5042624),\n",
       " ('whom', 'she', 0.5247077),\n",
       " ('often', 'she', 0.5113497),\n",
       " ('promoted', 'she', 0.51141727),\n",
       " ('brought', 'her', 0.51839703),\n",
       " ('initially', 'she', 0.53276813),\n",
       " ('franchise', 'she', 0.51567256),\n",
       " ('beginning', 'she', 0.52632385),\n",
       " ('sold', 'she', 0.51486236),\n",
       " ('pitch', 'she', 0.50092894),\n",
       " ('immediately', 'she', 0.5262939),\n",
       " ('picked', 'she', 0.5136176),\n",
       " ('stint', 'she', 0.50816613),\n",
       " ('phillies', 'she', 0.5243347),\n",
       " ('stood', 'she', 0.51328534),\n",
       " ('stayed', 'she', 0.5132445),\n",
       " ('whilst', 'she', 0.5259596),\n",
       " ('believed', 'she', 0.50342435),\n",
       " ('spending', 'she', 0.52002525),\n",
       " ('impressed', 'she', 0.50359756),\n",
       " ('arrived', 'she', 0.5200055),\n",
       " ('pistons', 'she', 0.5011432),\n",
       " ('claiming', 'she', 0.5047266),\n",
       " ('newton', 'she', 0.50049776),\n",
       " ('minors', 'she', 0.518329),\n",
       " ('duties', 'her', 0.50433046),\n",
       " ('occasionally', 'she', 0.5008765),\n",
       " ('wished', 'she', 0.50546896),\n",
       " ('wherever', 'porter42', 0.5453666),\n",
       " ('understands', 'sycv', 0.5126913),\n",
       " ('thrived', 'sycv', 0.5121255),\n",
       " ('maninredbandana', 'wilkinsboyle2009', 0.5305104),\n",
       " ('ucdavisaggies1', 'wilkinsboyle2009', 0.52210623),\n",
       " ('mrever', 'porter42', 0.5723449),\n",
       " ('bikeraceinfo1', 'wilkinsboyle2009', 0.51265454),\n",
       " ('efcprofile', 'nbcolympics1', 0.5111421),\n",
       " ('lifebehindcamera', 'wilkinsboyle2009', 0.5237975),\n",
       " ('mite', 'nbcolympics1', 0.50126016),\n",
       " ('nyteskenazi', 'porter42', 0.50645953),\n",
       " ('laweekly1', 'porter42', 0.5074456),\n",
       " ('walleghem1', 'wilkinsboyle2009', 0.64398706),\n",
       " ('chany1', 'porter42', 0.60785705),\n",
       " ('coutts', 'sycv', 0.5216865),\n",
       " ('jpress', 'wilkinsboyle2009', 0.5064631),\n",
       " ('jsonline1', 'nbcolympics1', 0.5370488),\n",
       " ('mtts', 'wilkinsboyle2009', 0.52615964),\n",
       " ('naseem', 'dorovskikh', 0.52766347),\n",
       " ('news13', 'porter42', 0.50106704),\n",
       " ('officialbiography', 'sycv', 0.5121208),\n",
       " ('persevered', 'sycv', 0.5380598),\n",
       " ('tights', 'porter42', 0.51199245),\n",
       " ('vaprus', 'wezemaal', 0.5007104),\n",
       " ('abc23', 'wilkinsboyle2009', 0.5576614),\n",
       " ('blackmore', 'porter42', 0.5251892),\n",
       " ('dn2804010', 'wilkinsboyle2009', 0.5275391),\n",
       " ('dvow1', 'porter42', 0.507223),\n",
       " ('forgave', 'sycv', 0.5528283),\n",
       " ('hjkltwop', 'porter42', 0.5014857),\n",
       " ('listens', 'sycv', 0.5119384),\n",
       " ('lotbb', 'lat890226', 0.5040006),\n",
       " ('nyt1998', 'porter42', 0.5331628),\n",
       " ('rotary9500', 'sycv', 0.51620704),\n",
       " ('thesetpieces', 'porter42', 0.5023137),\n",
       " ('alienated', 'porter42', 0.54671013),\n",
       " ('bankroll', 'sycv', 0.5912225),\n",
       " ('bbc19980909', 'porter42', 0.5499926),\n",
       " ('bbcksb161213', 'sycv', 0.5681827),\n",
       " ('biginterview', 'porter42', 0.56392694),\n",
       " ('defilippis', 'sycv', 0.50484717),\n",
       " ('fcsbat', 'sycv', 0.50774395),\n",
       " ('fixes', 'bhardwaj', 0.5067098),\n",
       " ('kozokin', 'lagging', 0.5326069),\n",
       " ('landsbergersteinitzpapers', 'porter42', 0.5000229),\n",
       " ('layden15may', 'sycv', 0.61584604),\n",
       " ('likeafricans', 'wilkinsboyle2009', 0.5864866),\n",
       " ('mdppsw', 'porter42', 0.50098246),\n",
       " ('mercurynews1', 'nbcolympics1', 0.5196123),\n",
       " ('philly1', 'nbcolympics1', 0.5015608),\n",
       " ('pqarchiver2002', 'sycv', 0.51975113),\n",
       " ('qiietlyhome', 'wilkinsboyle2009', 0.5555348),\n",
       " ('racingmuseum1', 'lat890226', 0.52077407),\n",
       " ('smithinterview', 'sycv', 0.52946234),\n",
       " ('sunsentinelturmoil', 'porter42', 0.505741),\n",
       " ('tpprof', 'sycv', 0.5112719),\n",
       " ('unicanberra', 'wilkinsboyle2009', 0.5873754),\n",
       " ('uslacrosse1', 'wilkinsboyle2009', 0.6219428),\n",
       " ('wallet', 'dorovskikh', 0.51915884),\n",
       " ('100random', 'sycv', 0.5106348),\n",
       " ('affirming', 'dorovskikh', 0.5328588),\n",
       " ('antoivat', 'sycv', 0.5742443),\n",
       " ('arse', 'sycv', 0.51915765),\n",
       " ('disposition', 'lagging', 0.5032666),\n",
       " ('fausports2', 'porter42', 0.5091014),\n",
       " ('favors', 'sycv', 0.53458905),\n",
       " ('gs020203', 'wilkinsboyle2009', 0.586246),\n",
       " ('hurdled', 'lagging', 0.50710225),\n",
       " ('lasbat', 'porter42', 0.5081586),\n",
       " ('massappeal1', 'wilkinsboyle2009', 0.519926),\n",
       " ('mercedes2010', 'wilkinsboyle2009', 0.5354146),\n",
       " ('more2008fallno', 'porter42', 0.5020747),\n",
       " ('officialbio', 'wilkinsboyle2009', 0.5015953),\n",
       " ('pfrbw', 'porter42', 0.51428187),\n",
       " ('relished', 'sycv', 0.6011399),\n",
       " ('rta', 'sycv', 0.53034395),\n",
       " ('skippered', 'sycv', 0.55362594),\n",
       " ('sunscoaster', 'wilkinsboyle2009', 0.5360463),\n",
       " ('uclabruins1', 'lat890226', 0.5478715),\n",
       " ('volleyhall1', 'wilkinsboyle2009', 0.55966264),\n",
       " ('wirtualnapolska', 'porter42', 0.5213794),\n",
       " ('ablbuzz137', 'sycv', 0.5314075),\n",
       " ('actatheltic', 'sycv', 0.51521534),\n",
       " ('aif0801', 'porter42', 0.60728276),\n",
       " ('arnoldstrongmanclassic1', 'sycv', 0.5390015),\n",
       " ('arrs8948', 'wilkinsboyle2009', 0.5177669),\n",
       " ('campbellnews', 'porter42', 0.5256819),\n",
       " ('corpse', 'dorovskikh', 0.5106842),\n",
       " ('coskun', 'sycv', 0.5102903),\n",
       " ('coupland', 'porter42', 0.50369513),\n",
       " ('fromklmore', 'lat890226', 0.5318104),\n",
       " ('gatherings', 'sycv', 0.57517374),\n",
       " ('guardian2007', 'sycv', 0.53771704),\n",
       " ('in141125', 'sycv', 0.512045),\n",
       " ('jhvonline9556', 'porter42', 0.5798681),\n",
       " ('lamming', 'sycv', 0.5068068),\n",
       " ('lanc', 'sycv', 0.51012605),\n",
       " ('liherald1', 'porter42', 0.57170457),\n",
       " ('nytimes2004', 'wilkinsboyle2009', 0.5159166),\n",
       " ('preached', 'sycv', 0.5074939),\n",
       " ('ref2', 'lewellen2004', 0.5108454),\n",
       " ('sn42007', 'wilkinsboyle2009', 0.50116146),\n",
       " ('swainepage20', 'porter42', 0.5332495),\n",
       " ('tafapsistbh', 'sycv', 0.5555395),\n",
       " ('theisraelproject1', 'wilkinsboyle2009', 0.5120807),\n",
       " ('thejewishweek2', 'wilkinsboyle2009', 0.56543636),\n",
       " ('tkpsu', 'sycv', 0.5005657),\n",
       " ('truus', 'rapide', 0.5145789),\n",
       " ('wenna', 'porter42', 0.5895032),\n",
       " ('windup', 'dorovskikh', 0.53868026),\n",
       " ('auk', 'sycv', 0.53556794),\n",
       " ('australianpolice', 'sycv', 0.6037976),\n",
       " ('biglongdiseasename', 'sycv', 0.5357094),\n",
       " ('blazeissue3', 'porter42', 0.5800016),\n",
       " ('coh', 'wilkinsboyle2009', 0.5588239),\n",
       " ('communblog', 'wilkinsboyle2009', 0.5330364),\n",
       " ('confortola', 'rapide', 0.51345545),\n",
       " ('dues', 'porter42', 0.508155),\n",
       " ('earlycontract', 'wilkinsboyle2009', 0.508629),\n",
       " ('ffksupporter1', 'porter42', 0.51359904),\n",
       " ('forbes2006', 'porter42', 0.5465144),\n",
       " ('gdcgjmiwtl', 'porter42', 0.5031566),\n",
       " ('gestured', 'sycv', 0.5312343),\n",
       " ('gfactfile', 'sycv', 0.5009193),\n",
       " ('google2768', 'wilkinsboyle2009', 0.5436084),\n",
       " ('gs101205', 'porter42', 0.5218307),\n",
       " ('gshm', 'sycv', 0.5806141),\n",
       " ('gvshf', 'wilkinsboyle2009', 0.606432),\n",
       " ('hanbin', 'porter42', 0.5647498),\n",
       " ('hoglan', 'upped', 0.5458724),\n",
       " ('iaaf2013', 'sycv', 0.52723014),\n",
       " ('ie03072005', 'sycv', 0.5576288),\n",
       " ('imbault', 'wezemaal', 0.5403048),\n",
       " ('joints', 'sycv', 0.5060589),\n",
       " ('jta130219', 'sycv', 0.55917317),\n",
       " ('landau124', 'ifscover', 0.5135489),\n",
       " ('latimes1994', 'sycv', 0.5280667),\n",
       " ('lvw', 'sycv', 0.5110319),\n",
       " ('nbastats', 'sycv', 0.59132034),\n",
       " ('nddaily', 'lat890226', 0.52490413),\n",
       " ('ndrhof', 'sycv', 0.5329843),\n",
       " ('nyt11', 'wilkinsboyle2009', 0.5000755),\n",
       " ('nyt981116', 'sycv', 0.50584763),\n",
       " ('nytimes1990', 'wilkinsboyle2009', 0.52587545),\n",
       " ('nzherald1', 'porter42', 0.5161139),\n",
       " ('oralhist1', 'sycv', 0.54319215),\n",
       " ('p235', 'wilkinsboyle2009', 0.5432393),\n",
       " ('patros', 'dorovskikh', 0.5398536),\n",
       " ('poundage', 'porter42', 0.508121),\n",
       " ('pqarchiver2004', 'porter42', 0.51236147),\n",
       " ('roundly', 'dorovskikh', 0.5108283),\n",
       " ('s2001w', 'porter42', 0.5067496),\n",
       " ('scott255', 'sycv', 0.5334303),\n",
       " ('sharpercollegecareer', 'porter42', 0.5231643),\n",
       " ('skillfully', 'porter42', 0.54270095),\n",
       " ('sme090125', 'porter42', 0.5172386),\n",
       " ('st060405', 'sycv', 0.5205882),\n",
       " ('stuff1', 'nbcolympics1', 0.5627073),\n",
       " ('surfs', 'porter42', 0.5429922),\n",
       " ('tailend', 'sycv', 0.5617783),\n",
       " ('tenthings', 'wilkinsboyle2009', 0.52083474),\n",
       " ('theplainsman1', 'sycv', 0.53555506),\n",
       " ('ulbio', 'sycv', 0.5726579),\n",
       " ('varies', 'sycv', 0.5876053),\n",
       " ('vastu', 'lagging', 0.5394073),\n",
       " ('wertheim20120211', 'porter42', 0.5038177),\n",
       " ('yesyes', 'sycv', 0.513808)]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(gsports.words[i[0]],gsports.words[i[1]], i[2]) for i in msports]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hockey', 'volleyball', 'soccer', 'basketball', 'football']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"football\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['presidents', 'presidential', 'clinton', 'vice', 'president']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"president\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([7186,  702, 1606, 2938,   71,   67,  808, 1749,  300,  787]),\n",
       " array([0.621892  , 0.63576126, 0.64200103, 0.6599604 , 0.7238898 ,\n",
       "        0.72987235, 0.737804  , 0.74939495, 0.80364037, 0.86988366],\n",
       "       dtype=float32))"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_nn_cos(glove[\"all\"].vector(\"she\") - glove[\"all\"].vector(\"he\") + glove[\"all\"].vector(\"man\"), glove[\"all\"].W)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W = glove[\"all\"].W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "m = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/madereth/anaconda3/envs/w266-env/lib/python3.6/site-packages/ipykernel_launcher.py:4: RuntimeWarning: invalid value encountered in true_divide\n",
      "  after removing the cwd from sys.path.\n",
      "/home/madereth/anaconda3/envs/w266-env/lib/python3.6/site-packages/ipykernel_launcher.py:6: RuntimeWarning: invalid value encountered in greater\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[  67],\n",
       "        [  71],\n",
       "        [ 787],\n",
       "        [ 808],\n",
       "        [1749],\n",
       "        [2003],\n",
       "        [2890],\n",
       "        [2938]]), array([[0.58033943],\n",
       "        [0.59222955],\n",
       "        [0.680215  ],\n",
       "        [0.5694386 ],\n",
       "        [0.57762533],\n",
       "        [0.52214944],\n",
       "        [0.5175899 ],\n",
       "        [0.53039855]], dtype=float32))"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_nn_cos_with_threshold(glove[\"all\"].vector(\"she\"),\n",
    "                          glove[\"all\"].vector(\"he\"),\n",
    "                          glove[\"all\"].vector(\"man\"),\n",
    "                          glove[\"all\"].W)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pregnant',\n",
       " 'actress',\n",
       " 'sister',\n",
       " 'herself',\n",
       " 'mother',\n",
       " 'girl',\n",
       " 'she',\n",
       " 'her',\n",
       " 'woman',\n",
       " 'man']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[glove[\"all\"].words[i] for i in [5508, 2890, 2003, 2938,  808, 1749,   67,   71,  787,  300]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "m += [1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cos_similarity(glove, word1, word2):\n",
    "    a = glove.vector(word1)\n",
    "    b = glove.vector(word2)\n",
    "    return np.dot(a,b) / (np.linalg.norm(a) * np.linalg.norm(b))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "correlates = [(\"cord\", \"smile\", 0.02),\n",
    "(\"rooster\", \"voyage\", 0.04),\n",
    "(\"noon\", \"string\", 0.04),\n",
    "(\"fruit\", \"furnace\", 0.05),\n",
    "(\"autograph\", \"shore\", 0.06),\n",
    "(\"automobile\", \"wizard\", 0.11),\n",
    "(\"mound\", \"stove\", 0.14),\n",
    "(\"grin\", \"implement\", 0.18),\n",
    "(\"asylum\", \"fruit\", 0.19),\n",
    "(\"asylum\", \"monk\", 0.39),\n",
    "(\"graveyard\", \"madhouse\", 0.42),\n",
    "(\"glass\", \"magician\", 0.44),\n",
    "(\"boy\", \"rooster\", 0.44),\n",
    "(\"cushion\", \"jewel\", 0.45),\n",
    "(\"monk\", \"slave\", 0.57),\n",
    "(\"asylum\", \"cemetary\", 0.79),\n",
    "(\"coast\", \"forest\", 0.85),\n",
    "(\"grin\", \"lad\", 0.88),\n",
    "(\"shore\", \"woodland\", 0.90),\n",
    "(\"monk\", \"oracle\", 0.91),\n",
    "(\"boy\", \"sage\", 0.96),\n",
    "(\"automobile\", \"cushion\", 0.97),\n",
    "(\"mound\", \"shore\", 0.97),\n",
    "(\"lad\", \"wizard\", 0.99),\n",
    "(\"forest\", \"graveyard\", 1.00),\n",
    "(\"food\", \"rooster\", 1.00),\n",
    "(\"cemetary\", \"woodland\", 1.18),\n",
    "(\"shore\", \"voyage\", 1.22),\n",
    "(\"bird\", \"woodland\", 1.24),\n",
    "(\"coast\", \"hill\", 1.26),\n",
    "(\"furnace\", \"implement\", 1.37),\n",
    "(\"crane\", \"rooster\", 1.41),\n",
    "(\"hill\", \"woodland\", 1.48),\n",
    "(\"car\", \"journey\", 1.55),\n",
    "(\"cemetary\", \"mound\", 1.69),\n",
    "(\"glass\", \"jewel\", 1.78),\n",
    "(\"magician\", \"oracle\", 1.82),\n",
    "(\"crane\", \"implement\", 2.37),\n",
    "(\"brother\", \"lad\", 2.41),\n",
    "(\"sage\", \"wizard\", 2.46),\n",
    "(\"oracle\", \"sage\", 2.61),\n",
    "(\"bird\", \"crane\", 2.63),\n",
    "(\"bird\", \"cock\", 2.63),\n",
    "(\"food\", \"fruit\", 2.69),\n",
    "(\"brother\", \"monk\", 2.74),\n",
    "(\"asylum\", \"madhouse\", 3.04),\n",
    "(\"furnace\", \"stove\", 3.11),\n",
    "(\"magician\", \"wizard\", 3.21),\n",
    "(\"hill\", \"mound\", 3.29),\n",
    "(\"cord\", \"string\", 3.41),\n",
    "(\"glass\", \"tumbler\", 3.45),\n",
    "(\"grin\", \"smile\", 3.46),\n",
    "(\"serf\", \"slave\", 3.46),\n",
    "(\"journey\", \"voyage\", 3.58),\n",
    "(\"autograph\", \"signature\", 3.59),\n",
    "(\"coast\", \"shore\", 3.60),\n",
    "(\"forest\", \"woodland\", 3.65),\n",
    "(\"implement\", \"tool\", 3.66),\n",
    "(\"cock\", \"rooster\", 3.68),\n",
    "(\"boy\", \"lad\", 3.82),\n",
    "(\"cushion\", \"pillow\", 3.81),\n",
    "(\"cemetary\", \"graveyard\", 3.88),\n",
    "(\"automobile\", \"car\", 3.92),\n",
    "(\"midday\", \"noon\", 3.94),\n",
    "(\"gem\", \"jewel\", 3.94)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('artists', 172878),\n",
       " ('monarch', 101333),\n",
       " ('programming', 17607),\n",
       " ('scientist', 81276),\n",
       " ('sports', 91223)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(topic,len(glove[topic].words)) for topic in topics]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.51533461"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cos_similarity(glove[\"all\"],correlates[-1][0],correlates[-1][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KendalltauResult(correlation=0.5071040191635332, pvalue=2.4900531333960791e-09)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.stats import kendalltau\n",
    "\n",
    "kendalltau([x[-1] for x in correlates],\n",
    "           [cos_similarity(glove[\"all\"],x[0],x[1]) for x in correlates])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def kendalltau_of_correlates(glove):\n",
    "    present = [c for c in correlates if c[0] in glove.words and c[1] in glove.words]\n",
    "\n",
    "    return len(present), kendalltau([x[-1] for x in present],\n",
    "               [cos_similarity(glove,x[0],x[1]) for x in present])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(65,\n",
       " KendalltauResult(correlation=0.5071040191635332, pvalue=2.4900531333960791e-09))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kendalltau_of_correlates(glove[\"all\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('artists',\n",
       "  (61,\n",
       "   KendalltauResult(correlation=0.37503490700589709, pvalue=2.0145283485225875e-05))),\n",
       " ('monarch',\n",
       "  (58,\n",
       "   KendalltauResult(correlation=0.2800855047151703, pvalue=0.0019379674654730476))),\n",
       " ('programming',\n",
       "  (19,\n",
       "   KendalltauResult(correlation=0.017543859649122806, pvalue=0.91641015594390629))),\n",
       " ('scientist',\n",
       "  (53,\n",
       "   KendalltauResult(correlation=0.23546536500461543, pvalue=0.012933316159810268))),\n",
       " ('sports',\n",
       "  (54,\n",
       "   KendalltauResult(correlation=0.083333517231030316, pvalue=0.37457513366530615)))]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(topic, kendalltau_of_correlates(glove[topic])) for topic in topics]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['heptathlete', 'she', 'hypo', 'heptathlon', 'decathlon']"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"sports\"].analogy(\"he\", \"she\", \"decathlon\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('artists', ['newlywed', 'researcher', 'battleships', 'meeco', 'programmer']),\n",
       " ('monarch', ['voiceover', 'arrangers', 'curls', 'nautch', 'programmer']),\n",
       " ('programming', ['engineer', 'goldman', 'her', 'she', 'programmer']),\n",
       " ('scientist', ['dancer', 'subculture', 'eniac', 'she', 'programmer']),\n",
       " ('sports', ['magnate', 'dorovskikh', 'gse', 'lagging', 'programmer'])]"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(topic, glove[topic].analogy(\"he\", \"she\", \"programmer\")) for topic in topics]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('artists', ['ennies', 'riesa', 'guardian6sept12', 'kruševac', 'decathlon']),\n",
       " ('sports', ['heptathlete', 'she', 'hypo', 'heptathlon', 'decathlon'])]"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(topic, glove[topic].analogy(\"he\", \"she\", \"decathlon\")) for topic in topics if \"decathlon\" in glove[topic].words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['software', 'designer', 'animator', 'programmers', 'programmer']"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"programmer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['prince', 'elizabeth', 'princess', 'queen', 'king']"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"king\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['medallist', 'medalist', 'bronze', 'gold', 'silver']"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"silver\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('artists', ['sister', 'she', 'her', 'queen', 'king']),\n",
       " ('monarch', ['husband', 'she', 'her', 'queen', 'king']),\n",
       " ('programming', ['lady', 'luther', 'countess', 'lovelace', 'king']),\n",
       " ('scientist', ['husband', 'narai', 'her', 'she', 'king']),\n",
       " ('sports', ['billie', 'she', 'her', 'fahd', 'king'])]"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(topic, glove[topic].analogy(\"he\", \"she\", \"king\")) for topic in topics]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('artists', ['she', 'her', 'amateur', 'shoot', 'boxing']),\n",
       " ('monarch',\n",
       "  ['amateur', 'featherweight', 'lineal', 'cruiserweight', 'boxing']),\n",
       " ('programming', ['voodoo', 'symphonic', 'l3', 'pentagram', 'boxing']),\n",
       " ('scientist', ['herself', 'ywca', 'conciliatory', 'gandhian', 'boxing']),\n",
       " ('sports', ['middleweight', 'amateur', 'she', 'her', 'boxing'])]"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(topic, glove[topic].analogy(\"he\", \"she\", \"boxing\")) for topic in topics]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gnome', 'mariadb', 'osi', 'python', 'eclipse']"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"programming\"].analogy(\"java\", \"eclipse\", \"python\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['oracle', 'settled', 'microsystems', 'sun', 'basic']"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"programming\"].analogy(\"java\", \"sun\", \"basic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['science', 'mathematics', 'biology', 'chemistry', 'physics']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"physics\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['she', 'woman', 'mother', 'nurse', 'doctor']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"doctor\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['degree', 'doctorate', 'her', 'she', 'doctor']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"scientist\"].analogy(\"he\", \"she\", \"doctor\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['harvard', 'university', 'sociology', 'lecturer', 'professor']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"professor\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['waitress', 'welder', 'electrician', 'hairdresser', 'plumber']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"plumber\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['oncologist', 'urologist', 'gynecologist', 'pediatrician', 'dermatologist']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"dermatologist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['psychiatrist', 'cardiologist', 'physician', 'neurologist', 'pediatrician']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"she\", \"he\", \"pediatrician\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['oncologist', 'urologist', 'gynecologist', 'pediatrician', 'cardiologist']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"cardiologist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['oncologist', 'gynecologist', 'pediatrician', 'dermatologist', 'urologist']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove[\"all\"].analogy(\"he\", \"she\", \"urologist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def projections(W, v):\n",
    "    return np.dot(W, v) / np.linalg.norm(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def projections_onto_he_she_diff(g):\n",
    "    return projections(g.W, g.vector(\"she\") - g.vector(\"he\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.034944884"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(projections(glove[\"all\"].W, glove[\"all\"].vector(\"she\") - glove[\"all\"].vector(\"he\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.16739798,  0.12225122,  0.13215803, ...,  0.11365189,\n",
       "        0.06668267,  0.06668267], dtype=float32)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.abs(projections(glove[\"all\"].W, glove[\"all\"].vector(\"she\") - glove[\"all\"].vector(\"he\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.072163448"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(np.abs(projections_onto_he_she_diff(glove[\"all\"])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('artists', 0.06692563),\n",
       " ('monarch', 0.079268932),\n",
       " ('programming', 0.075883418),\n",
       " ('scientist', 0.066126578),\n",
       " ('sports', 0.078573577)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(topic, np.mean(np.abs(projections_onto_he_she_diff(glove[topic])))) for topic in topics]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}