demos/more_examples/simple/MarvelTutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# PyGraphistry Example: Graphing the Marvel Universe\n",
    "### Plots hero social network based on co-appearences between heroes\n",
    "**Install: `pip install \"graphistry[igraph]\"`**\n",
    "\n",
    "Note: `pip install igraph` is the wrong package. if installing manually, use `python-igraph`\n",
    "\n",
    "*  Uses pandas, igraph, and PyGraphistry\n",
    "*  Combines comic book and hero data\n",
    "*  Near end, computes clusters and to avoid a hairball, weakens the edge weights between  nodes of different clusters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from io import open\n",
    "import pandas as pd\n",
    "import igraph # Install Igraph with pip install python-igraph\n",
    "import graphistry\n",
    "\n",
    "#graphistry.register(key='MY_API_KEY', server='labs.graphistry.com') #https://www.graphistry.com/api-request"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Load heroes, comics, appearences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Heroes: 6486\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hero_id</th>\n",
       "      <th>hero_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>24-HOUR MAN/EMMANUEL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3-D MAN/CHARLES CHANDLER &amp; HAROLD CHANDLER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>4-D MAN/MERCURIO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hero_id                                   hero_name\n",
       "0        1                        24-HOUR MAN/EMMANUEL\n",
       "1        2  3-D MAN/CHARLES CHANDLER & HAROLD CHANDLER\n",
       "2        3                            4-D MAN/MERCURIO"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('../../data/characters.txt', encoding=\"latin-1\") as f:\n",
    "    lines = f.readlines()\n",
    "heroes = pd.DataFrame(\n",
    "    list(map(lambda x: (int(x.split(':')[0].split(' ')[1]), x.split(': ', 1)[1].split('\\n')[0]), lines)),\n",
    "    columns=['hero_id', 'hero_name'])\n",
    "print('#Heroes:', len(heroes))\n",
    "heroes[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Comics:  12942\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comic_id</th>\n",
       "      <th>comic_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6487</td>\n",
       "      <td>AA2 35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6488</td>\n",
       "      <td>M/PRM 35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6489</td>\n",
       "      <td>M/PRM 36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   comic_id comic_name\n",
       "0      6487     AA2 35\n",
       "1      6488   M/PRM 35\n",
       "2      6489   M/PRM 36"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('../../data/comics.txt', encoding=\"latin-1\") as f:\n",
    "    lines = f.readlines()\n",
    "comics = pd.DataFrame(\n",
    "    list(map(lambda x: (int(x.split(':')[0].split(' ')[1]), x.split(': ', 1)[1].split('\\n')[0]), lines)),\n",
    "    columns=['comic_id', 'comic_name'])\n",
    "print('#Comics: ', len(comics))\n",
    "comics[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hero</th>\n",
       "      <th>comic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>6488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>6489</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hero  comic\n",
       "0     1   6487\n",
       "1     2   6488\n",
       "2     2   6489"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('../../data/appearances.txt', encoding=\"latin-1\") as f:\n",
    "    lines = f.readlines()[len(heroes) + len(comics) + 2:]\n",
    "def expand (line):\n",
    "    parts = list(map(int, line.split(' ')))\n",
    "    return [(parts[0], role) for role in parts[1:]]\n",
    "appearences = pd.DataFrame(\n",
    "    [item for sublist in list(map(expand, lines)) for item in sublist],\n",
    "    columns=['hero', 'comic'])\n",
    "appearences[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "source": [
    "#Link heroes who co-appear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#edges 168267\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hero_x</th>\n",
       "      <th>hero_y</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1999</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6459</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6459</td>\n",
       "      <td>1999</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hero_x  hero_y  counts\n",
       "0    1999       1       1\n",
       "1    6459       1       1\n",
       "2    6459    1999       1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# You may need to install numexpr: pip install numexpr\n",
    "coappearences = \\\n",
    "    appearences\\\n",
    "        .merge(appearences, on='comic')\\\n",
    "        .merge(comics, left_on='comic', right_on='comic_id')\\\n",
    "        [['hero_x', 'hero_y']]\\\n",
    "        .query('hero_x > hero_y')\n",
    "unique_coappearences = coappearences.drop_duplicates(['hero_x', 'hero_y']).set_index(['hero_x', 'hero_y'])\n",
    "unique_coappearences['counts'] = coappearences.groupby(['hero_x', 'hero_y']).size()\n",
    "unique_coappearences = unique_coappearences.reset_index()\n",
    "print('#edges', len(unique_coappearences))        \n",
    "unique_coappearences[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Plot!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "g = graphistry.bind(source='hero_x', destination='hero_y', edge_title='counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <iframe id=\"4ff21543-22f1-4d3a-b899-441d41b70660\" src=\"https://labs.graphistry.com/graph/graph.html?dataset=PyGraphistry/NWIRCQDK36&type=vgraph&viztoken=78a2ae09a25296a2b2731f2811c0747d78b8cfd7&usertag=86f11264-pygraphistry-0.9.63&splashAfter=1554615641&info=true\"\n",
       "                    allowfullscreen=\"true\" webkitallowfullscreen=\"true\" mozallowfullscreen=\"true\"\n",
       "                    oallowfullscreen=\"true\" msallowfullscreen=\"true\"\n",
       "                    style=\"width:100%; height:500px; border: 1px solid #DDD\">\n",
       "            </iframe>\n",
       "        \n",
       "            <script>\n",
       "                $(\"#4ff21543-22f1-4d3a-b899-441d41b70660\").bind('mousewheel', function(e) {\n",
       "                e.preventDefault();\n",
       "                });\n",
       "            </script>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g.plot(unique_coappearences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Label Nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Here we are using two dataframes, one for edges and one for nodes\n",
    "g2 = g.bind(node='hero_id', point_title='hero_name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <iframe id=\"48854c81-2742-44ff-b342-057ae7fae595\" src=\"https://labs.graphistry.com/graph/graph.html?dataset=PyGraphistry/L0QXF3147B&type=vgraph&viztoken=586112fcd160082d0dfe0077f24f1716233665b1&usertag=86f11264-pygraphistry-0.9.63&splashAfter=1554615650&info=true\"\n",
       "                    allowfullscreen=\"true\" webkitallowfullscreen=\"true\" mozallowfullscreen=\"true\"\n",
       "                    oallowfullscreen=\"true\" msallowfullscreen=\"true\"\n",
       "                    style=\"width:100%; height:500px; border: 1px solid #DDD\">\n",
       "            </iframe>\n",
       "        \n",
       "            <script>\n",
       "                $(\"#48854c81-2742-44ff-b342-057ae7fae595\").bind('mousewheel', function(e) {\n",
       "                e.preventDefault();\n",
       "                });\n",
       "            </script>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g2.plot(unique_coappearences, heroes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Color using igraph infomap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Infomap Community Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#clusters 212\n"
     ]
    }
   ],
   "source": [
    "#Warning: slow\n",
    "ig = g2.pandas2igraph(unique_coappearences, directed=False)\n",
    "clusters = ig.community_infomap()\n",
    "(i_edges, i_nodes) = g2.igraph2pandas(ig)\n",
    "print('#clusters', str(len(list(set(clusters.membership)))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#colored nodes 6467\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>denseid</th>\n",
       "      <th>cluster</th>\n",
       "      <th>hero_id</th>\n",
       "      <th>hero_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>1999</td>\n",
       "      <td>FROST, CARMILLA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>24-HOUR MAN/EMMANUEL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>6459</td>\n",
       "      <td>G'RATH</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   denseid  cluster  hero_id             hero_name\n",
       "0        0       32     1999       FROST, CARMILLA\n",
       "1        1       32        1  24-HOUR MAN/EMMANUEL\n",
       "2        2       32     6459                G'RATH"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_colored = pd.DataFrame({'cluster': clusters.membership})\\\n",
    "    .reset_index().rename(columns={'index': 'denseid'})\\\n",
    "    .merge(i_nodes.reset_index().rename(columns={'index':'denseid'}), on='denseid')\\\n",
    "    .merge(heroes, left_on='hero_id', right_on='hero_id')\n",
    "print('#colored nodes', str(len(nodes_colored)))\n",
    "nodes_colored[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cluster</th>\n",
       "      <th>denseid</th>\n",
       "      <th>hero_id</th>\n",
       "      <th>hero_name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>color</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24</td>\n",
       "      <td>1626</td>\n",
       "      <td>1626</td>\n",
       "      <td>1626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>1171</td>\n",
       "      <td>1171</td>\n",
       "      <td>1171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24</td>\n",
       "      <td>890</td>\n",
       "      <td>890</td>\n",
       "      <td>890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>574</td>\n",
       "      <td>574</td>\n",
       "      <td>574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24</td>\n",
       "      <td>490</td>\n",
       "      <td>490</td>\n",
       "      <td>490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>23</td>\n",
       "      <td>468</td>\n",
       "      <td>468</td>\n",
       "      <td>468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>449</td>\n",
       "      <td>449</td>\n",
       "      <td>449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>23</td>\n",
       "      <td>413</td>\n",
       "      <td>413</td>\n",
       "      <td>413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>23</td>\n",
       "      <td>386</td>\n",
       "      <td>386</td>\n",
       "      <td>386</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       cluster  denseid  hero_id  hero_name\n",
       "color                                      \n",
       "0           24     1626     1626       1626\n",
       "1           24     1171     1171       1171\n",
       "2           24      890      890        890\n",
       "3           24      574      574        574\n",
       "4           24      490      490        490\n",
       "5           23      468      468        468\n",
       "6           23      449      449        449\n",
       "7           23      413      413        413\n",
       "8           23      386      386        386"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_colored['color'] = nodes_colored.apply(lambda x: x['cluster'] % 9, axis=1)\n",
    "nodes_colored.pivot_table(index=['color'], aggfunc=lambda x: len(x.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "g3 = g2.bind(point_color='color', edge_weight='counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <iframe id=\"18436df6-75bc-4e35-ae65-34ee09d3c3f3\" src=\"https://labs.graphistry.com/graph/graph.html?dataset=PyGraphistry/VA3FO6GDVJ&type=vgraph&viztoken=5072e353c628eedb84716d222ae567be841281d2&usertag=86f11264-pygraphistry-0.9.63&splashAfter=1554615746&info=true\"\n",
       "                    allowfullscreen=\"true\" webkitallowfullscreen=\"true\" mozallowfullscreen=\"true\"\n",
       "                    oallowfullscreen=\"true\" msallowfullscreen=\"true\"\n",
       "                    style=\"width:100%; height:500px; border: 1px solid #DDD\">\n",
       "            </iframe>\n",
       "        \n",
       "            <script>\n",
       "                $(\"#18436df6-75bc-4e35-ae65-34ee09d3c3f3\").bind('mousewheel', function(e) {\n",
       "                e.preventDefault();\n",
       "                });\n",
       "            </script>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g3.plot(unique_coappearences,  nodes_colored)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Restrict to biggest communities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# big clusters 10\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cluster</th>\n",
       "      <th>cluster_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>535</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cluster  cluster_size\n",
       "0        0          1260\n",
       "1        1           820\n",
       "2        2           535"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "big_clusters = nodes_colored\\\n",
    "    .pivot_table(index=['cluster'], aggfunc=lambda x: len(x.unique()))\\\n",
    "    .rename(columns={'hero_id': 'cluster_size'})\\\n",
    "    .query('cluster_size > 100')\\\n",
    "    .reset_index()[['cluster', 'cluster_size']]\n",
    "print('# big clusters', len(big_clusters))\n",
    "big_clusters[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# nodes 3612\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>denseid</th>\n",
       "      <th>cluster</th>\n",
       "      <th>hero_id</th>\n",
       "      <th>hero_name</th>\n",
       "      <th>color</th>\n",
       "      <th>cluster_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2186</td>\n",
       "      <td>GORILLA-MAN</td>\n",
       "      <td>0</td>\n",
       "      <td>1260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3-D MAN/CHARLES CHANDLER &amp; HAROLD CHANDLER</td>\n",
       "      <td>0</td>\n",
       "      <td>1260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>2555</td>\n",
       "      <td>HUMAN ROBOT</td>\n",
       "      <td>0</td>\n",
       "      <td>1260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   denseid  cluster  hero_id                                   hero_name  \\\n",
       "0        6        0     2186                                 GORILLA-MAN   \n",
       "1        7        0        2  3-D MAN/CHARLES CHANDLER & HAROLD CHANDLER   \n",
       "2        8        0     2555                                 HUMAN ROBOT   \n",
       "\n",
       "   color  cluster_size  \n",
       "0      0          1260  \n",
       "1      0          1260  \n",
       "2      0          1260  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "good_nodes = nodes_colored.merge(big_clusters, on='cluster')\n",
    "print('# nodes', len(good_nodes))\n",
    "good_nodes[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# edges 114648\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hero_x</th>\n",
       "      <th>hero_y</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2186</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2555</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3491</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hero_x  hero_y  counts\n",
       "0    2186       2       3\n",
       "1    2555       2       3\n",
       "2    3491       2       3"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "good_edges = unique_coappearences\\\n",
    "    .merge(good_nodes, left_on='hero_x', right_on='hero_id')\\\n",
    "    .merge(good_nodes, left_on='hero_y', right_on='hero_id')\\\n",
    "    [['hero_x', 'hero_y', 'counts']]\n",
    "print('# edges', len(good_edges))\n",
    "good_edges[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <iframe id=\"7d2eb9ce-ae15-45a7-84b4-a0bfce380d34\" src=\"https://labs.graphistry.com/graph/graph.html?dataset=PyGraphistry/AWU4BUXH6E&type=vgraph&viztoken=df6e807e828103526a0cfe532ad3ce3a32c460ef&usertag=86f11264-pygraphistry-0.9.63&splashAfter=1554615753&info=true\"\n",
       "                    allowfullscreen=\"true\" webkitallowfullscreen=\"true\" mozallowfullscreen=\"true\"\n",
       "                    oallowfullscreen=\"true\" msallowfullscreen=\"true\"\n",
       "                    style=\"width:100%; height:500px; border: 1px solid #DDD\">\n",
       "            </iframe>\n",
       "        \n",
       "            <script>\n",
       "                $(\"#7d2eb9ce-ae15-45a7-84b4-a0bfce380d34\").bind('mousewheel', function(e) {\n",
       "                e.preventDefault();\n",
       "                });\n",
       "            </script>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g3.plot(good_edges, good_nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Seperate communities\n",
    "### Treat intra-community edges as strong edge weights, and inter-community as weak edge weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hero_x</th>\n",
       "      <th>hero_y</th>\n",
       "      <th>counts</th>\n",
       "      <th>is_inner</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2186</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2555</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3491</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hero_x  hero_y  counts  is_inner  weight\n",
       "0    2186       2       3      True      10\n",
       "1    2555       2       3      True      10\n",
       "2    3491       2       3      True      10"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#label edges whether they stay inside a cluster or connect nodes in different clusters\n",
    "good_edges2 = good_edges\\\n",
    "        .merge(\\\n",
    "                 good_nodes[['cluster', 'hero_id']].rename(columns={'cluster': 'cluster_x'}),\\\n",
    "                 left_on='hero_x', right_on='hero_id')\\\n",
    "        .merge(\\\n",
    "                 good_nodes[['cluster', 'hero_id']].rename(columns={'cluster': 'cluster_y'}),\\\n",
    "                 left_on='hero_y', right_on='hero_id')\n",
    "good_edges2['is_inner'] = good_edges2.apply(lambda x: x['cluster_x'] == x['cluster_y'], axis=1)\n",
    "\n",
    "#bind to edge_weight\n",
    "good_edges2['weight'] = good_edges2.apply(lambda x: 10 if x['is_inner'] else 8, axis=1)\n",
    "good_edges2 = good_edges2[['hero_x', 'hero_y', 'counts', 'is_inner', 'weight']]\n",
    "good_edges2[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Plot; control the edge weight in the settings panel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <iframe id=\"a4056cda-7153-4a0f-9465-2c68e56c6730\" src=\"https://labs.graphistry.com/graph/graph.html?dataset=PyGraphistry/W74X67VPJU&type=vgraph&viztoken=f1a669f3c55e15757880190bfdc981751dc48c2f&usertag=86f11264-pygraphistry-0.9.63&splashAfter=1554615767&info=true\"\n",
       "                    allowfullscreen=\"true\" webkitallowfullscreen=\"true\" mozallowfullscreen=\"true\"\n",
       "                    oallowfullscreen=\"true\" msallowfullscreen=\"true\"\n",
       "                    style=\"width:100%; height:500px; border: 1px solid #DDD\">\n",
       "            </iframe>\n",
       "        \n",
       "            <script>\n",
       "                $(\"#a4056cda-7153-4a0f-9465-2c68e56c6730\").bind('mousewheel', function(e) {\n",
       "                e.preventDefault();\n",
       "                });\n",
       "            </script>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g3.bind(edge_weight='weight').plot(good_edges2, good_nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Filter by k-core shell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#shells 98\n"
     ]
    }
   ],
   "source": [
    "shells = ig.shell_index()\n",
    "print('#shells', str(len(list(set(shells)))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#shelled nodes 6467\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>denseid</th>\n",
       "      <th>shell</th>\n",
       "      <th>cluster</th>\n",
       "      <th>hero_id</th>\n",
       "      <th>hero_name</th>\n",
       "      <th>color</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>32</td>\n",
       "      <td>1999</td>\n",
       "      <td>FROST, CARMILLA</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>24-HOUR MAN/EMMANUEL</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>6459</td>\n",
       "      <td>G'RATH</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   denseid  shell  cluster  hero_id             hero_name  color\n",
       "0        0     13       32     1999       FROST, CARMILLA      5\n",
       "1        1      5       32        1  24-HOUR MAN/EMMANUEL      5\n",
       "2        2      5       32     6459                G'RATH      5"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_shelled = pd.DataFrame({'shell': shells})\\\n",
    "    .reset_index().rename(columns={'index': 'denseid'})\\\n",
    "    .merge(nodes_colored, on='denseid')\n",
    "print('#shelled nodes', str(len(nodes_shelled)))\n",
    "nodes_shelled[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Plot: Use the histogram tool to filter for the smaller shells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <iframe id=\"e0252a05-e016-4e73-b7b8-268ab1853e68\" src=\"https://labs.graphistry.com/graph/graph.html?dataset=PyGraphistry/E7498J51D9&type=vgraph&viztoken=e918994fa8542ecc5e9a8b4404e26c59316311b6&usertag=86f11264-pygraphistry-0.9.63&splashAfter=1554615777&info=true\"\n",
       "                    allowfullscreen=\"true\" webkitallowfullscreen=\"true\" mozallowfullscreen=\"true\"\n",
       "                    oallowfullscreen=\"true\" msallowfullscreen=\"true\"\n",
       "                    style=\"width:100%; height:500px; border: 1px solid #DDD\">\n",
       "            </iframe>\n",
       "        \n",
       "            <script>\n",
       "                $(\"#e0252a05-e016-4e73-b7b8-268ab1853e68\").bind('mousewheel', function(e) {\n",
       "                e.preventDefault();\n",
       "                });\n",
       "            </script>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g3.plot(unique_coappearences,  nodes_shelled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}