Web scraping using Python(Beautifulsoup).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retriving Job title, location in Monter.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "URL = 'https://www.monster.com/jobs/search/?q=Software-Developer&where=Australia'\n",
    "page = requests.get(URL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(page.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(page.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = soup.find(id='ResultsContainer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(results.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "job_elems = results.find_all('section', class_='card-content')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Performance Engineer, Software Systems\n",
      "Magic Leap, Inc.\n",
      "Plantation, FL; Sunnyvale, CA; Culver New York City, CA; Seattle, WA; Austin, TX; Toronto, NY\n",
      "\n",
      "SQL BI (SSRS, SSIS) developer for Blackboard - NYC\n",
      "LanceSoft Inc\n",
      "New york, WA\n",
      "\n",
      "Python Developer\n",
      "LanceSoft Inc\n",
      "Woodlands, WA\n",
      "\n",
      "Junior QA Analyst - Melbourne, Victoria\n",
      "Mediaocean\n",
      "Melbourne, VIC\n",
      "\n",
      "Payroll Tester\n",
      "Dialog Group\n",
      "Brisbane, QLD\n",
      "\n",
      "Test Analyst\n",
      "Dialog Group\n",
      "Canberra, ACT\n",
      "\n",
      "Customer Experience Technical Analyst - Sydney, New South Wales\n",
      "Mediaocean\n",
      "Sydney, NSW\n",
      "\n",
      "Account Manager\n",
      "Dialog Group\n",
      "Canberra, ACT\n",
      "\n",
      "Test Analyst / Senior Test Analyst\n",
      "Dialog Group\n",
      "Melbourne, VIC\n",
      "\n",
      "Senior Practice Manager - IES (WA)\n",
      "Blue Ocean Ventures\n",
      "New York, WA\n",
      "\n",
      "Software Developer/Senior Software Developer\n",
      "Beacon Hill Staffing Group, LLC\n",
      "Green Bay, WI\n",
      "\n",
      "Enterprise Account Executive\n",
      "Zuora\n",
      "Melbourne, VIC\n",
      "\n",
      "Associate Strategist- Social Media\n",
      "Khoros, LLC\n",
      "Sydney, FL\n",
      "\n",
      "Performance Test Consultant\n",
      "Dialog Group\n",
      "Melbourne, VIC\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for job_elem in job_elems:\n",
    "    title_elem = job_elem.find('h2', class_='title')\n",
    "    company_elem = job_elem.find('div', class_='company')\n",
    "    location_elem = job_elem.find('div', class_='location')\n",
    "    if None in (title_elem, company_elem, location_elem):\n",
    "        continue\n",
    "    print(title_elem.text.strip())\n",
    "    print(company_elem.text.strip())\n",
    "    print(location_elem.text.strip())\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding a job by title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "python_jobs = results.find_all('h2', string='Python Developer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "python_jobs = results.find_all('h2',\n",
    "                               string=lambda text: 'python' in text.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    " print(len(python_jobs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python Developer\n",
      "Apply here: https://job-openings.monster.com/python-developer-woodlands-wa-us-lancesoft-inc/4755ec59-d0db-4ce9-8385-b4df7c1e9f7c\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for p_job in python_jobs:\n",
    "    link = p_job.find('a')['href']\n",
    "    print(p_job.text.strip())\n",
    "    print(f\"Apply here: {link}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IMBD Web Scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get(\"http://www.imdb.com/search/title?release_date=2017&sort=num_votes,desc&page=1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "<!DOCTYPE html>\n",
      "<html\n",
      "    xmlns:og=\"http://ogp.me/ns#\"\n",
      "    xmlns:fb=\"http://www.facebook.com/2008/fbml\">\n",
      "    <head>\n",
      "         \n",
      "        <meta charset=\"utf-8\">\n",
      "        <meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n",
      "\n",
      "    <meta name=\"apple-itunes-app\" content=\"app-id=342792525, app-argument=imdb:///?src=mdot\">\n",
      "\n",
      "\n",
      "\n",
      "        <script type=\"text/javascript\">var IMDbTimer={starttime: new Date().getTime(),pt:'java'};</script>\n",
      "\n",
      "<script>\n",
      "    if (typeof uet == 'function') {\n",
      "      uet(\"bb\", \"LoadTitle\"\n"
     ]
    }
   ],
   "source": [
    "print(response.text[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bs4.BeautifulSoup"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "html_soup = BeautifulSoup(response.text, 'html.parser')\n",
    "type(html_soup)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading movie details in 1st page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_containers = html_soup.find_all('div', class_ = 'lister-item mode-advanced')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'bs4.element.ResultSet'>\n"
     ]
    }
   ],
   "source": [
    "print(type(movie_containers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n"
     ]
    }
   ],
   "source": [
    "print(len(movie_containers))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retriving movie detials of first movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_movie = movie_containers[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<div class=\"lister-top-right\">\n",
       "<div class=\"ribbonize\" data-caller=\"filmosearch\" data-tconst=\"tt3315342\"></div>\n",
       "</div>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_movie.div"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a href=\"/title/tt3315342/\"> <img alt=\"Logan\" class=\"loadlate\" data-tconst=\"tt3315342\" height=\"98\" loadlate=\"https://m.media-amazon.com/images/M/MV5BYzc5MTU4N2EtYTkyMi00NjdhLTg3NWEtMTY4OTEyMzJhZTAzXkEyXkFqcGdeQXVyNjc1NTYyMjg@._V1_UX67_CR0,0,67,98_AL_.jpg\" src=\"https://m.media-amazon.com/images/G/01/imdb/images/nopicture/large/film-184890147._CB466725069_.png\" width=\"67\"/>\n",
       "</a>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_movie.a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<h3 class=\"lister-item-header\">\n",
       "<span class=\"lister-item-index unbold text-primary\">1.</span>\n",
       "<a href=\"/title/tt3315342/\">Logan</a>\n",
       "<span class=\"lister-item-year text-muted unbold\">(2017)</span>\n",
       "</h3>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_movie.h3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a href=\"/title/tt3315342/\">Logan</a>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_movie.h3.a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Logan'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_movie.h3.a.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_year = first_movie.find('span', class_ = \"lister-item-year text-muted unbold\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function str.split(sep=None, maxsplit=-1)>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_year.text.split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'8.1'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_movie.strong.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_imdb = float(first_movie.strong.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.1"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_imdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77\n"
     ]
    }
   ],
   "source": [
    "first_mscore = first_movie.find('span', class_ = 'metascore favorable')\n",
    "first_mscore = int(first_mscore.text)\n",
    "print(first_mscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<span data-value=\"617330\" name=\"nv\">617,330</span>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_votes = first_movie.find('span', attrs = {'name':'nv'})\n",
    "first_votes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "617330\n"
     ]
    }
   ],
   "source": [
    "print(int(first_votes['data-value']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'137 min'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#runtime\n",
    "first_runtime = first_movie.find('span', class_ = \"runtime\").text\n",
    "first_runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Action, Drama, Sci-Fi'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#genre\n",
    "first_genre = first_movie.find('span', class_ = \"genre\").text\n",
    "first_genre[1:].strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bs4.element.Tag"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eighth_movie_mscore = movie_containers[6].find('div', class_ = 'ratings-metascore')\n",
    "type(eighth_movie_mscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = [str(i) for i in range(1,5)]\n",
    "years_url = [str(i) for i in range(2000, 2018)]   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = []\n",
    "years = []\n",
    "votes = []\n",
    "metascores = []\n",
    "imdb_ratings = []\n",
    "genres = []\n",
    "runtimes = []\n",
    "\n",
    "for year_url in years_url:\n",
    "    for page in pages:\n",
    "        page_html = BeautifulSoup(response.text, 'html.parser')\n",
    "        mv_containers = page_html.find_all('div', class_ = 'lister-item mode-advanced')\n",
    "        for container in mv_containers:\n",
    "            if container.find('div', class_ = 'ratings-metascore') is not None:\n",
    "#name   \n",
    "                name = container.h3.a.text\n",
    "                names.append(name)\n",
    "#year\n",
    "                year = container.find('span', class_ = \"lister-item-year\").text\n",
    "                years.append(year)\n",
    "#rating\n",
    "                rating = float(container.strong.text)\n",
    "                imdb_ratings.append(rating)\n",
    "#votes\n",
    "                vote = container.find('span', attrs = {'name':'nv'})['data-value']\n",
    "                votes.append(int(vote))\n",
    "#metascore\n",
    "                score = container.find('span', class_ = 'metascore').text\n",
    "                metascores.append(int(score))\n",
    "#genre\n",
    "                genre = container.find('span', class_ = \"genre\").text.strip()\n",
    "                genres.append(genre)\n",
    "#runtime\n",
    "                runtime = container.find('span', class_ = \"runtime\").text\n",
    "                runtimes.append(runtime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie</th>\n",
       "      <th>Year</th>\n",
       "      <th>IMDB</th>\n",
       "      <th>Metascore</th>\n",
       "      <th>Vote</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Logan</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>8.1</td>\n",
       "      <td>77</td>\n",
       "      <td>617330</td>\n",
       "      <td>137 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Thor: Ragnarok</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>7.9</td>\n",
       "      <td>74</td>\n",
       "      <td>553312</td>\n",
       "      <td>130 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Guardians of the Galaxy Vol. 2</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>7.6</td>\n",
       "      <td>67</td>\n",
       "      <td>540913</td>\n",
       "      <td>136 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Star Wars: Episode VIII - The Last Jedi</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84</td>\n",
       "      <td>540316</td>\n",
       "      <td>152 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Wonder Woman</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>7.4</td>\n",
       "      <td>76</td>\n",
       "      <td>528536</td>\n",
       "      <td>141 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3019</th>\n",
       "      <td>Bright</td>\n",
       "      <td>(I) (2017)</td>\n",
       "      <td>6.3</td>\n",
       "      <td>29</td>\n",
       "      <td>161957</td>\n",
       "      <td>117 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3020</th>\n",
       "      <td>Darkest Hour</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>7.4</td>\n",
       "      <td>75</td>\n",
       "      <td>161611</td>\n",
       "      <td>125 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3021</th>\n",
       "      <td>Valerian and the City of a Thousand Planets</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>6.5</td>\n",
       "      <td>51</td>\n",
       "      <td>155052</td>\n",
       "      <td>137 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3022</th>\n",
       "      <td>Baywatch</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>5.5</td>\n",
       "      <td>37</td>\n",
       "      <td>153550</td>\n",
       "      <td>116 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3023</th>\n",
       "      <td>American Made</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>7.2</td>\n",
       "      <td>65</td>\n",
       "      <td>150610</td>\n",
       "      <td>115 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3024 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Movie        Year  IMDB  \\\n",
       "0                                           Logan      (2017)   8.1   \n",
       "1                                  Thor: Ragnarok      (2017)   7.9   \n",
       "2                  Guardians of the Galaxy Vol. 2      (2017)   7.6   \n",
       "3         Star Wars: Episode VIII - The Last Jedi      (2017)   7.0   \n",
       "4                                    Wonder Woman      (2017)   7.4   \n",
       "...                                           ...         ...   ...   \n",
       "3019                                       Bright  (I) (2017)   6.3   \n",
       "3020                                 Darkest Hour      (2017)   7.4   \n",
       "3021  Valerian and the City of a Thousand Planets      (2017)   6.5   \n",
       "3022                                     Baywatch      (2017)   5.5   \n",
       "3023                                American Made      (2017)   7.2   \n",
       "\n",
       "      Metascore    Vote  Runtime                      Genre  \n",
       "0            77  617330  137 min  Action, Biography, Comedy  \n",
       "1            74  553312  130 min  Action, Biography, Comedy  \n",
       "2            67  540913  136 min  Action, Biography, Comedy  \n",
       "3            84  540316  152 min  Action, Biography, Comedy  \n",
       "4            76  528536  141 min  Action, Biography, Comedy  \n",
       "...         ...     ...      ...                        ...  \n",
       "3019         29  161957  117 min  Action, Biography, Comedy  \n",
       "3020         75  161611  125 min  Action, Biography, Comedy  \n",
       "3021         51  155052  137 min  Action, Biography, Comedy  \n",
       "3022         37  153550  116 min  Action, Biography, Comedy  \n",
       "3023         65  150610  115 min  Action, Biography, Comedy  \n",
       "\n",
       "[3024 rows x 7 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.DataFrame({'Movie':names, 'Year': years, 'IMDB': imdb_ratings, 'Metascore': metascores, 'Vote':votes, 'Runtime' : runtimes, 'Genre':genre})\n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie</th>\n",
       "      <th>Year</th>\n",
       "      <th>IMDB</th>\n",
       "      <th>Metascore</th>\n",
       "      <th>Vote</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3019</th>\n",
       "      <td>Bright</td>\n",
       "      <td>(I) (2017)</td>\n",
       "      <td>6.3</td>\n",
       "      <td>29</td>\n",
       "      <td>161957</td>\n",
       "      <td>117 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3020</th>\n",
       "      <td>Darkest Hour</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>7.4</td>\n",
       "      <td>75</td>\n",
       "      <td>161611</td>\n",
       "      <td>125 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3021</th>\n",
       "      <td>Valerian and the City of a Thousand Planets</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>6.5</td>\n",
       "      <td>51</td>\n",
       "      <td>155052</td>\n",
       "      <td>137 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3022</th>\n",
       "      <td>Baywatch</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>5.5</td>\n",
       "      <td>37</td>\n",
       "      <td>153550</td>\n",
       "      <td>116 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3023</th>\n",
       "      <td>American Made</td>\n",
       "      <td>(2017)</td>\n",
       "      <td>7.2</td>\n",
       "      <td>65</td>\n",
       "      <td>150610</td>\n",
       "      <td>115 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Movie        Year  IMDB  \\\n",
       "3019                                       Bright  (I) (2017)   6.3   \n",
       "3020                                 Darkest Hour      (2017)   7.4   \n",
       "3021  Valerian and the City of a Thousand Planets      (2017)   6.5   \n",
       "3022                                     Baywatch      (2017)   5.5   \n",
       "3023                                American Made      (2017)   7.2   \n",
       "\n",
       "      Metascore    Vote  Runtime                      Genre  \n",
       "3019         29  161957  117 min  Action, Biography, Comedy  \n",
       "3020         75  161611  125 min  Action, Biography, Comedy  \n",
       "3021         51  155052  137 min  Action, Biography, Comedy  \n",
       "3022         37  153550  116 min  Action, Biography, Comedy  \n",
       "3023         65  150610  115 min  Action, Biography, Comedy  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = test_df[['Movie', 'Year', 'IMDB', 'Metascore', 'Vote', 'Runtime', 'Genre']]\n",
    "test_df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning Year column to remove extra characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.loc[:, 'Year'] = test_df['Year'].str[-5:-1].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie</th>\n",
       "      <th>Year</th>\n",
       "      <th>IMDB</th>\n",
       "      <th>Metascore</th>\n",
       "      <th>Vote</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Logan</td>\n",
       "      <td>2017</td>\n",
       "      <td>8.1</td>\n",
       "      <td>77</td>\n",
       "      <td>617330</td>\n",
       "      <td>137 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Thor: Ragnarok</td>\n",
       "      <td>2017</td>\n",
       "      <td>7.9</td>\n",
       "      <td>74</td>\n",
       "      <td>553312</td>\n",
       "      <td>130 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Guardians of the Galaxy Vol. 2</td>\n",
       "      <td>2017</td>\n",
       "      <td>7.6</td>\n",
       "      <td>67</td>\n",
       "      <td>540913</td>\n",
       "      <td>136 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Star Wars: Episode VIII - The Last Jedi</td>\n",
       "      <td>2017</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84</td>\n",
       "      <td>540316</td>\n",
       "      <td>152 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Wonder Woman</td>\n",
       "      <td>2017</td>\n",
       "      <td>7.4</td>\n",
       "      <td>76</td>\n",
       "      <td>528536</td>\n",
       "      <td>141 min</td>\n",
       "      <td>Action, Biography, Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Movie  Year  IMDB  Metascore    Vote  \\\n",
       "0                                    Logan  2017   8.1         77  617330   \n",
       "1                           Thor: Ragnarok  2017   7.9         74  553312   \n",
       "2           Guardians of the Galaxy Vol. 2  2017   7.6         67  540913   \n",
       "3  Star Wars: Episode VIII - The Last Jedi  2017   7.0         84  540316   \n",
       "4                             Wonder Woman  2017   7.4         76  528536   \n",
       "\n",
       "   Runtime                      Genre  \n",
       "0  137 min  Action, Biography, Comedy  \n",
       "1  130 min  Action, Biography, Comedy  \n",
       "2  136 min  Action, Biography, Comedy  \n",
       "3  152 min  Action, Biography, Comedy  \n",
       "4  141 min  Action, Biography, Comedy  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Writing Data to CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.to_csv('movie_rating.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}