notebooks/2_0_mayra_preprocessing-pca.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "O3AKE9uDpnhl"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "from sklearn.pipeline import FeatureUnion, Pipeline\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jMCqNLiZpnh3"
   },
   "outputs": [],
   "source": [
    "#Read the data using the Unnamed (probably id) as index\n",
    "#url = 'https://s3.amazonaws.com/drivendata/data/4/public/81e8f2de-9915-4934-b9ae-9705685c9d50.csv'\n",
    "url = '../src/data/raw/training.csv'\n",
    "training = pd.read_csv(url, index_col='Unnamed: 0')\n",
    "\n",
    "labels = ['Function', 'Object_Type', 'Operating_Status', 'Position_Type', 'Pre_K', 'Reporting', \n",
    "          'Sharing', 'Student_Type', 'Use']\n",
    "\n",
    "numeric = ['FTE', 'Total']\n",
    "\n",
    "categoric = [ 'Facility_or_Department', 'Function_Description', \n",
    "            'Fund_Description', 'Job_Title_Description', 'Location_Description', \n",
    "            'Object_Description', 'Position_Extra', 'Program_Description', 'SubFund_Description', \n",
    "            'Sub_Object_Description', \n",
    "            'Text_1', 'Text_2', 'Text_3', 'Text_4']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_PxtgWq6pniC"
   },
   "source": [
    "## Data cleaning\n",
    "\n",
    "### Part 1: numeric data\n",
    "First of all, the numeric columns `FTE` and `Total` must be cleaned. The outliers will be treated as `NaN` values and imputation will be applied to the missing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LKH5j3vqpniH"
   },
   "outputs": [],
   "source": [
    "# Remove inconsistent data\n",
    "training[training['FTE'] < 0] = np.nan\n",
    "training[training['Total'] < 0] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 178
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6515,
     "status": "ok",
     "timestamp": 1553782546223,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "4LFeigU0pniT",
    "outputId": "e25bc3da-d3ac-49a9-851f-24b7fe9f309e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    125849.000000\n",
       "mean          0.427436\n",
       "std           0.573810\n",
       "min           0.000000\n",
       "25%           0.001078\n",
       "50%           0.133337\n",
       "75%           1.000000\n",
       "max          46.800000\n",
       "Name: FTE, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training['FTE'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 99
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6471,
     "status": "ok",
     "timestamp": 1553782546233,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "fihg4CaYpnio",
    "outputId": "c2676c6b-a5bc-498f-b054-d40917de1edb"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function</th>\n",
       "      <th>Use</th>\n",
       "      <th>Sharing</th>\n",
       "      <th>Reporting</th>\n",
       "      <th>Student_Type</th>\n",
       "      <th>Position_Type</th>\n",
       "      <th>Object_Type</th>\n",
       "      <th>Pre_K</th>\n",
       "      <th>Operating_Status</th>\n",
       "      <th>Object_Description</th>\n",
       "      <th>...</th>\n",
       "      <th>Sub_Object_Description</th>\n",
       "      <th>Location_Description</th>\n",
       "      <th>FTE</th>\n",
       "      <th>Function_Description</th>\n",
       "      <th>Facility_or_Department</th>\n",
       "      <th>Position_Extra</th>\n",
       "      <th>Total</th>\n",
       "      <th>Program_Description</th>\n",
       "      <th>Fund_Description</th>\n",
       "      <th>Text_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Function, Use, Sharing, Reporting, Student_Type, Position_Type, Object_Type, Pre_K, Operating_Status, Object_Description, Text_2, SubFund_Description, Job_Title_Description, Text_3, Text_4, Sub_Object_Description, Location_Description, FTE, Function_Description, Facility_or_Department, Position_Extra, Total, Program_Description, Fund_Description, Text_1]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 25 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training[training['FTE'] < 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 178
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6437,
     "status": "ok",
     "timestamp": 1553782546236,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "TxVhbCGBpni6",
    "outputId": "e3a0aca8-11e3-4dbb-d361-d88539756fac"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    3.518450e+05\n",
       "mean     1.696423e+04\n",
       "std      3.402292e+05\n",
       "min      0.000000e+00\n",
       "25%      1.384847e+02\n",
       "50%      6.696800e+02\n",
       "75%      4.999610e+03\n",
       "max      1.297000e+08\n",
       "Name: Total, dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training['Total'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 73
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1097,
     "status": "ok",
     "timestamp": 1553784053004,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "xsFqr4Z5vYv_",
    "outputId": "dbf3c0e0-570e-48c6-8175-cecf6a10534a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:58: DeprecationWarning: Class Imputer is deprecated; Imputer was deprecated in version 0.20 and will be removed in 0.22. Import impute.SimpleImputer from sklearn instead.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "imp = Imputer(strategy=\"median\")\n",
    "\n",
    "training[numeric] = imp.fit_transform(training[numeric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 826,
     "status": "ok",
     "timestamp": 1553784074442,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "nQf8KlrQvZNZ",
    "outputId": "69a9ebd4-578e-4104-896e-43702710d27f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FTE      0\n",
       "Total    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training[numeric].isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fySNphkGpnjV"
   },
   "source": [
    "### Part 2: Categoric data\n",
    "\n",
    "The strings must be normalized: symbols removed, all letters to lower case. Then, they can be transformed to numeric data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "C3Ta-ushpnjk"
   },
   "outputs": [],
   "source": [
    "for category in categoric:\n",
    "    training[category] = training[category].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 285
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8224,
     "status": "ok",
     "timestamp": 1553782548051,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "2SsELzlOpnjx",
    "outputId": "5b4fcc75-4e2b-4a23-d4be-9aed8ac228bb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Facility_or_Department     176\n",
       "Function_Description       655\n",
       "Fund_Description           141\n",
       "Job_Title_Description     3455\n",
       "Location_Description       352\n",
       "Object_Description         570\n",
       "Position_Extra             580\n",
       "Program_Description        418\n",
       "SubFund_Description        267\n",
       "Sub_Object_Description     159\n",
       "Text_1                    1389\n",
       "Text_2                     279\n",
       "Text_3                      35\n",
       "Text_4                     240\n",
       "dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Check if the unique values decrease\n",
    "training[categoric].nunique(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bcFlsgHOpnkF"
   },
   "outputs": [],
   "source": [
    "#Convert columns\n",
    "training[categoric] = training[categoric].apply(lambda x: x.astype('object'), axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 356
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9735,
     "status": "ok",
     "timestamp": 1553782549586,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "UvfEWHVJpnkU",
    "outputId": "3c332206-449a-47d0-dd1f-2c1e74624fad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 400277 entries, 134338 to 415831\n",
      "Data columns (total 14 columns):\n",
      "Facility_or_Department    53186 non-null object\n",
      "Function_Description      298676 non-null object\n",
      "Fund_Description          179157 non-null object\n",
      "Job_Title_Description     272317 non-null object\n",
      "Location_Description      155504 non-null object\n",
      "Object_Description        332143 non-null object\n",
      "Position_Extra            244180 non-null object\n",
      "Program_Description       280943 non-null object\n",
      "SubFund_Description       263731 non-null object\n",
      "Sub_Object_Description    70748 non-null object\n",
      "Text_1                    251891 non-null object\n",
      "Text_2                    87431 non-null object\n",
      "Text_3                    108613 non-null object\n",
      "Text_4                    53384 non-null object\n",
      "dtypes: object(14)\n",
      "memory usage: 45.8+ MB\n"
     ]
    }
   ],
   "source": [
    "training[categoric].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4W1BXPTSpnkj"
   },
   "source": [
    "Before removing symbols, let's take a peek on some values and so we can detect the diferrences later\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 142
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9716,
     "status": "ok",
     "timestamp": 1553782549595,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "1i-cbqwDpnk3",
    "outputId": "89bd11a0-53dd-4856-dd0b-ddbf611a71a7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "non-project                        True\n",
       "instruction                        True\n",
       "unalloc budgets/schools            True\n",
       "basic (fefp k-12)                 False\n",
       "employee retirement               False\n",
       "ela e-teaching sheltered eng      False\n",
       "Name: Function_Description, dtype: bool"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_functions = training['Function_Description'].value_counts(normalize=True) > 0.05\n",
    "some_functions.head(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2Z7HZPFTpnlL"
   },
   "outputs": [],
   "source": [
    "# Remove symbols\n",
    "'''for category in categoric:\n",
    "    training[category] = training[category].str.replace(r\"[&/\\)('\\\"-.,]\",\"\")'''\n",
    "vec = HashingVectorizer(token_pattern=\"[A-Za-z0-9]+(?=\\\\s+)\", ngram_range=(1,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 142
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9704,
     "status": "ok",
     "timestamp": 1553782549609,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "DTUN_VN8pnla",
    "outputId": "6f8e5853-d169-4457-ef57-583cf438dcd5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "non-project                       0.191686\n",
       "instruction                       0.128882\n",
       "unalloc budgets/schools           0.055458\n",
       "basic (fefp k-12)                 0.044473\n",
       "employee retirement               0.043723\n",
       "ela e-teaching sheltered eng      0.021093\n",
       "Name: Function_Description, dtype: float64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training['Function_Description'].value_counts(normalize=True).head(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6XHA4bcapnmE"
   },
   "outputs": [],
   "source": [
    "training[categoric] = training[categoric].fillna(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10588,
     "status": "ok",
     "timestamp": 1553782550520,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "M6Dp_7IBpnmk",
    "outputId": "d2052a8b-99ab-491e-90a3-5a40f3e17222"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training['Function_Description'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "58JZmJ9BpnnO"
   },
   "outputs": [],
   "source": [
    "# Join all text items in a row that have a space in between\n",
    "text_data = training[categoric].apply(lambda x: \" \".join(x), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1125
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 19280,
     "status": "ok",
     "timestamp": 1553782559233,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "XKMbWcOxpnny",
    "outputId": "4acb00f5-4b90-4413-8fec-d8001e84d9ed"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "134338      general fund teacher-elementary    kindergar...\n",
       "206341     rgn  gob  (blank)  contractor services undesi...\n",
       "326408      general purpose school tcher 2nd grade  pers...\n",
       "364634     unalloc budgets/schools  teacher, short term ...\n",
       "47683      non-project  teacher, secondary (high)   teac...\n",
       "229958                                                     \n",
       "417668      local fund   educational supplies and materi...\n",
       "126378     non-project  sub manager, food service  distr...\n",
       "275539     ela s - teaching spanish only  teacher, eleme...\n",
       "85262      unalloc budgets/schools  teacher,retrd shrt t...\n",
       "304569                                                     \n",
       "330504     instruction general fund   supplies  primary ...\n",
       "84272     position control pools  general purpose school...\n",
       "64760                                                      \n",
       "21870      basic (fefp k-12)              conversion cha...\n",
       "18698     all campus payroll instruction general operati...\n",
       "169454     basic (fefp k-12)              general fund  ...\n",
       "169914     security services general admin lieutenant co...\n",
       "189701     non-project  (blank)  electricity undesignate...\n",
       "43727                                                      \n",
       "5614       direction of support services - pupils * spec...\n",
       "291539     unalloc budgets/schools  teacher, short term ...\n",
       "307038     disadvantaged youth * title i - disadvantaged...\n",
       "27645      guidance services              conversion cha...\n",
       "126388      general purpose school   personal services -...\n",
       "14962       school federal projects tcher english (secon...\n",
       "84040                                                      \n",
       "61639      core matters  teacher, short term sub   emplo...\n",
       "266302     ela e-teaching sheltered eng  teacher, elemen...\n",
       "225768    transportation department transportation gener...\n",
       "                                ...                        \n",
       "333748     psychological services * general psychologist...\n",
       "292626    new construction department administration fac...\n",
       "343458     unalloc budgets/schools  teacher, super sub d...\n",
       "128261     special ed. - severe needs  teacher,spec ed c...\n",
       "186318     ela s - teaching spanish only  teacher, eleme...\n",
       "210652    teacher supply  school federal projects   pers...\n",
       "340354     title iii  ela  ela general assignment   empl...\n",
       "435205     read to achieve round three  teacher, element...\n",
       "174827     title i  ela general assignment   salaries of...\n",
       "277504                                                     \n",
       "248062    vocational curriculum development and instruct...\n",
       "70455      basic (fefp k-12)              conversion cha...\n",
       "203453     staff services                 extended learn...\n",
       "58170       school federal projects   personal services ...\n",
       "72072       general purpose school assistant principal -...\n",
       "68311      title ii-part a-teacher qualit  teacher, elem...\n",
       "225892     ela e-teaching sheltered eng  teacher, elemen...\n",
       "446383                                                     \n",
       "155111    transportation department transportation gener...\n",
       "278248     instruction general fund   supplies  primary ...\n",
       "397424     ns  gob  secondary spec ed para   additional/...\n",
       "375092      general purpose school   personal services -...\n",
       "220181    child nutrition food services (child nutrition...\n",
       "307423                                                     \n",
       "46691      title ii-part a-teacher qualit  teacher, elem...\n",
       "109283     inst staff training svcs       general fund  ...\n",
       "102430     title ii,d  teacher,retrd shrt term sub   sal...\n",
       "413949      schoolwide schools school liaison   parent/t...\n",
       "433672     non-project  library technician ii  ed resour...\n",
       "415831    instruction and curriculum instruction \"title ...\n",
       "Length: 400277, dtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bCZg8h_tpnoJ"
   },
   "outputs": [],
   "source": [
    "hec = HashingVectorizer(token_pattern=\"[A-Za-z0-9]+(?=\\\\s+)\", norm=None, binary=False,\n",
    "                                                     ngram_range=(1,2))\n",
    "hashed_text = hec.fit_transform(text_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 142
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 677,
     "status": "ok",
     "timestamp": 1553782581281,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "5tB5b3XRpnoS",
    "outputId": "2d1b16ea-d006-4066-bb18-552cd7b60b83"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10866634 entries, 0 to 10866633\n",
      "Data columns (total 1 columns):\n",
      "0    float64\n",
      "dtypes: float64(1)\n",
      "memory usage: 82.9 MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "hashed_df = pd.DataFrame(hashed_text.data)\n",
    "print(hashed_df.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 740,
     "status": "ok",
     "timestamp": 1553782595840,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "P6vD-pT-pnom",
    "outputId": "66eb22f1-1705-491d-a319-0480cbfd1e88"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0\n",
       "0  1.0\n",
       "1  2.0\n",
       "2 -1.0\n",
       "3 -1.0\n",
       "4  1.0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashed_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QJOUTSzfub7W"
   },
   "source": [
    "### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2503,
     "status": "ok",
     "timestamp": 1553784106317,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "PCCJQkgJqKrQ",
    "outputId": "a4965ff4-d5da-4209-eb42-f20c3caff51a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py:3790: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  downcast=downcast, **kwargs)\n"
     ]
    },
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-3911cd516b3a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     42\u001b[0m     ])\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m \u001b[0mpreprocessed_sparse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    296\u001b[0m         \"\"\"\n\u001b[1;32m    297\u001b[0m         \u001b[0mlast_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_final_estimator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 298\u001b[0;31m         \u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfit_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    299\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlast_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'fit_transform'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    300\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mlast_step\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    228\u001b[0m                 Xt, fitted_transformer = fit_transform_one_cached(\n\u001b[1;32m    229\u001b[0m                     \u001b[0mcloned_transformer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 230\u001b[0;31m                     **fit_params_steps[name])\n\u001b[0m\u001b[1;32m    231\u001b[0m                 \u001b[0;31m# Replace the transformer of the step with the fitted\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m                 \u001b[0;31m# transformer. This is necessary when loading the transformer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/externals/joblib/memory.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    340\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 342\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcall_and_shelve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit_transform_one\u001b[0;34m(transformer, X, y, weight, **fit_params)\u001b[0m\n\u001b[1;32m    612\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_fit_transform_one\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    613\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'fit_transform'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 614\u001b[0;31m         \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    615\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    616\u001b[0m         \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/base.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    460\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0my\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    461\u001b[0m             \u001b[0;31m# fit method of arity 1 (unsupervised transformation)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 462\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    463\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    464\u001b[0m             \u001b[0;31m# fit method of arity 2 (supervised transformation)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/preprocessing/_function_transformer.py\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    158\u001b[0m                           DeprecationWarning)\n\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkw_args\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkw_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0minverse_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'deprecated'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/preprocessing/_function_transformer.py\u001b[0m in \u001b[0;36m_transform\u001b[0;34m(self, X, y, func, kw_args)\u001b[0m\n\u001b[1;32m    199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         return func(X, *((y,) if pass_y else ()),\n\u001b[0;32m--> 201\u001b[0;31m                     **(kw_args if kw_args else {}))\n\u001b[0m",
      "\u001b[0;32m<ipython-input-44-3911cd516b3a>\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mget_numeric_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFunctionTransformer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnumeric\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0mget_dense\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFunctionTransformer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtodense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m pl = Pipeline([\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/sparse/base.py\u001b[0m in \u001b[0;36mtodense\u001b[0;34m(self, order, out)\u001b[0m\n\u001b[1;32m    847\u001b[0m             \u001b[0;31m`\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatrix\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mobject\u001b[0m \u001b[0mthat\u001b[0m \u001b[0mshares\u001b[0m \u001b[0mthe\u001b[0m \u001b[0msame\u001b[0m \u001b[0mmemory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    848\u001b[0m         \"\"\"\n\u001b[0;32m--> 849\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masmatrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    850\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    851\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/sparse/compressed.py\u001b[0m in \u001b[0;36mtoarray\u001b[0;34m(self, order, out)\u001b[0m\n\u001b[1;32m    960\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0morder\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    961\u001b[0m             \u001b[0morder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_swap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 962\u001b[0;31m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_toarray_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    963\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_contiguous\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_contiguous\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Output array must be C or F contiguous'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/sparse/base.py\u001b[0m in \u001b[0;36m_process_toarray_args\u001b[0;34m(self, order, out)\u001b[0m\n\u001b[1;32m   1185\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1186\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1187\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1188\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#preprocessed_df = pd.concat([training[numeric],hashed_df], axis='columns') \n",
    "\n",
    "# Define combine_text_columns()\n",
    "def combine_text_columns(data_frame):\n",
    "    \"\"\" converts all text in each row of data_frame to single vector \"\"\"\n",
    "    \n",
    "    # Drop non-text columns that are in the df\n",
    "    text_data = data_frame[categoric]\n",
    "    \n",
    "    # Replace nans with blanks\n",
    "    text_data.fillna(\"\", inplace=True)\n",
    "    \n",
    "    # Join all text items in a row that have a space in between\n",
    "    return text_data.apply(lambda x: \" \".join(x), axis=1)\n",
    "\n",
    "# Preprocess the text data: get_text_data\n",
    "get_text_data = FunctionTransformer(combine_text_columns, validate=False)\n",
    "\n",
    "# Preprocess the numeric data: get_numeric_data\n",
    "get_numeric_data = FunctionTransformer(lambda x: x[numeric], validate=False)\n",
    "\n",
    "get_dense = FunctionTransformer(lambda x: x.todense(), validate=False)\n",
    "\n",
    "pl = Pipeline([\n",
    "        ('union', FeatureUnion(\n",
    "            transformer_list = [\n",
    "                ('numeric_features', Pipeline([\n",
    "                    ('selector', get_numeric_data)\n",
    "                ])),\n",
    "                ('text_features', Pipeline([\n",
    "                    ('selector', get_text_data),\n",
    "                    ('vectorizer',HashingVectorizer(token_pattern=\"[A-Za-z0-9]+(?=\\\\s+)\", \n",
    "                                                    norm=None, \n",
    "                                                    binary=False,\n",
    "                                                    ngram_range=(1,2)) \n",
    "                    )\n",
    "                ]))\n",
    "             ]\n",
    "        )),\n",
    "        ('to_dense', get_dense),\n",
    "        ('pca', PCA(n_components=2))\n",
    "    ])\n",
    "\n",
    "preprocessed_sparse = pl.fit_transform(training)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-40-c6aed783f2ec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpreprocesse_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreprocessed_sparse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/sparse/compressed.py\u001b[0m in \u001b[0;36mtoarray\u001b[0;34m(self, order, out)\u001b[0m\n\u001b[1;32m    960\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0morder\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    961\u001b[0m             \u001b[0morder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_swap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 962\u001b[0;31m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_toarray_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    963\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_contiguous\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_contiguous\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Output array must be C or F contiguous'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/sparse/base.py\u001b[0m in \u001b[0;36m_process_toarray_args\u001b[0;34m(self, order, out)\u001b[0m\n\u001b[1;32m   1185\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1186\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1187\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1188\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "preprocesses_df = pd.DataFrame(preprocessed_sparse.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zFFmXWoZv-It"
   },
   "outputs": [],
   "source": [
    "preprocessed_df[numeric] = imp.fit_transform(preprocessed_df[numeric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1260,
     "status": "ok",
     "timestamp": 1553784203333,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "e3B9IKu3wOWt",
    "outputId": "d04a145c-52af-48a7-bfa4-59ce02f0cf44"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FTE</th>\n",
       "      <th>Total</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.133337</td>\n",
       "      <td>669.680000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.133337</td>\n",
       "      <td>669.680000</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>141.579900</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1166.760000</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>33668.537368</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        FTE         Total    0\n",
       "0  0.133337    669.680000  1.0\n",
       "1  0.133337    669.680000  2.0\n",
       "2  0.000000    141.579900 -1.0\n",
       "3  0.000000   1166.760000 -1.0\n",
       "4  1.000000  33668.537368  1.0"
      ]
     },
     "execution_count": 35,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tnizfmBWwZYK"
   },
   "outputs": [],
   "source": [
    "cov_matrix = preprocessed_df.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 365
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 944,
     "status": "ok",
     "timestamp": 1553785804150,
     "user": {
      "displayName": "Mayra Dantas",
      "photoUrl": "https://lh5.googleusercontent.com/-vROongVa7tk/AAAAAAAAAAI/AAAAAAAAAxk/lY0ACgMVguw/s64/photo.jpg",
      "userId": "17106868938271974483"
     },
     "user_tz": 180
    },
    "id": "2uyAOnXQ2aLg",
    "outputId": "9f44024c-f890-4bbf-9241-6c66edaea5ab"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f508b6e5470>"
      ]
     },
     "execution_count": 42,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcQAAAFKCAYAAACD5S+3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFnJJREFUeJzt3X9sVnf99/HnuToqPwp8KdK5MBAE\nAloGgvzYaNQsKwlBXWJucEUysowRYcyZbSpYE5HFsmrExTmjDKffZEHW1btmbDHUH8OYsFqm045W\nJtJBYSOBXtlgKwItvc79h1lvy9ZCL865Lk55PpKT9FzXuc55/8WL9+d8PucEYRiGSJJ0jUvluwBJ\nkq4GBqIkSRiIkiQBBqIkSYCBKEkSYCBKkgTAdXFfYG0wKe5LKE8eO/WXfJegmGQKR+S7BMVk6LBh\nsZ37Sv69/2l4JKoyshZ7IEqSrg0FQb4ruDIOmUqShB2iJCkiBUGyW0QDUZIUiaQPmRqIkqRI2CFK\nkoQdoiRJgB2iJElA8jtEl11IkoQdoiQpIg6ZSpJE8occDURJUiTsECVJIvmTagxESVIkkt4hJn3I\nV5KkSNghSpIi4ZCpJEkkf8jUQJQkRcIOUZIk7BAlSQLsECVJApIfiC67kCQJO0RJUkS8hyhJEskf\nMjUQJUmRsEOUJIl4O8QtW7bQ1NREEARUVlYya9asnu927NjBrl27SKVSzJw5k29+85tZXcNAlCRF\nIq4Ocd++fbS1tVFTU0NrayuVlZXU1NQA0NHRwZNPPslvf/tbrrvuOu6++27+/ve/8/GPf3zA1zEQ\nJUmRiKtDbGhooLy8HIApU6Zw+vRpOjo6KCoqYsiQIQwZMoR///vfDB8+nLNnzzJ69OisruOyC0nS\nVS2dTjNmzJie/eLiYtrb2wH4wAc+wPr16ykvL+fWW29l9uzZTJ48OavrGIiSpEgUBEHW20CEYdjz\nd0dHB9u2bWP37t384Q9/oKmpiVdffTWr+g1ESVIkUkGQ9dafkpIS0ul0z/7JkycZN24cAK2trUyY\nMIHi4mIKCwuZN28ezc3N2dWf1a8kSbpIUBBkvfWnrKyM+vp6AFpaWigpKaGoqAiA8ePH09rayrlz\n5wBobm5m0qRJWdXvpBpJUiRSMc2qmTt3LqWlpVRUVBAEAZs2baKuro6RI0eyePFiVq9ezapVqygo\nKGDOnDnMmzcvq+sE4X8PxsZgbTApztMrjx479Zd8l6CYZApH5LsExWTosGGxnXv3lDlZ/3ZJ698i\nrCQ7doiSpEhcaujzauc9REmSsEOUJEUkrnuIuWIgSpIiEaSSPehoIEqSImGHKEkSyZ9UYyBKkiIR\nFDhkKklS4odMkx3nkiRFxA5RkhSJIJXsDtFAlCRFIuU9REmSnGUqSRKQ/EDst79tbGzstd/Z2dnz\nd21tbTwVSZISKVWQynq7GvRbxY9//ONe+/fcc0/P388991w8FUmSEimuFwTnSr+BePGrEv97P+bX\nKEqSlFP93kMMgqDP/Yu/kyRd21KDedlFJpPh3LlzPd3gu/uZTIZMJpOTAiVJyTCoH912/PhxPvOZ\nz/QaHn133w5RkvTfkv7otn4Dsb6+niFDhuSqFklSgl0tk2Oy1W9/u3r16lzVIUlKuKAglfV2Nei3\nQ3QmqSTpcg3qIdNDhw7xla98pc/vf/jDH0ZekCRJ+dBvIF5//fWsXLkyV7VIkhJsUL/tYtSoUSxY\nsCBXtUiSEuxqeQRbtvqtfv/+/b32t23bFmsxkqTkGtSPbps5c2av/b1798ZajCQpuQb1LNOLF987\n61SS1JcgdXUEW7YG9D5En04jSepL0u8h9huIzc3NLFu2DPhPd3j48GGWLVvW8+i2X/3qVzkpUpKk\nuPUbiL7zUJJ0ua6We4HZ6jcQx48fn6s6JEkJN6gDUZKky3VNTaqRJKkvQUFBvku4IgaiJCkSDplK\nkgSkEj5kmuzqJUmKiB2iJCkSDplKkoSBKEkS4LILSZIAO0RJkgADUZIkIPlvu0h29ZIkRcQOUZIU\nCSfVSJKE9xAlSQIMREmSAIdMJUkCIOXrnyRJSv6QabKrlyQpIgaiJCkSQUEq6+1StmzZwh133EFF\nRQWvvPLK+x6zdetW7rzzzqzrd8hUkhSJuCbV7Nu3j7a2NmpqamhtbaWyspKamppexxw6dIiXXnqJ\nIUOGZH0dO0RJUiTi6hAbGhooLy8HYMqUKZw+fZqOjo5ex1RXV/PAAw9cUf0GoiQpEnEFYjqdZsyY\nMT37xcXFtLe39+zX1dWxYMECxo8ff0X1O2QqSYpErtYhhmHY8/epU6eoq6vjF7/4BSdOnLii8xqI\nkqRIBKl41iGWlJSQTqd79k+ePMm4ceMA+POf/8ybb77JypUr6ezs5OjRo2zZsoXKysoBX8chU0nS\nVa2srIz6+noAWlpaKCkpoaioCIAlS5bwm9/8hmeeeYbHH3+c0tLSrMIQ7BAlSVGJqUOcO3cupaWl\nVFRUEAQBmzZtoq6ujpEjR7J48eLIrhOE/z0YG4O1waQ4T688euzUX/JdgmKSKRyR7xIUk6HDhsV2\n7n//3+9n/dvh/+erEVaSHTtESVIkAp9lKkkSsQ2Z5oqBKEmKhoEoSVLy34eY7OolSYpI7B2iMxEH\nr/v/Z16+S1BMHj37ar5LUBI5ZCpJEgaiJEmQ/HuIBqIkKRp2iJIkYSBKkgTJf1JNsgd8JUmKiB2i\nJCkaTqqRJAnvIUqSBBAYiJIk4ZCpJElghyhJ0n8kPBCT3d9KkhQRO0RJUjS8hyhJUvKfVGMgSpKi\nkfB7iAaiJCkaBqIkSb4PUZKk/0h4h5jsOJckKSJ2iJKkaATJ7rEMRElSNAxESZIgNBAlScIOUZIk\nAIIg3xVcEQNRkhSNhK9DTHb1kiRFxA5RkhQJJ9VIkgROqpEkCTAQJUkCDERJksB7iJIk/UfCAzHZ\n1UuSFBE7RElSNHxSjSRJJH7I1ECUJEXCSTWSJEHin2VqIEqSomGHKEkSiQ/EZFcvSVJE7BAlSdFI\neIdoIEqSIuEsU0mSwA5RkiTAJ9VIkgTE2iFu2bKFpqYmgiCgsrKSWbNm9Xz34osv8oMf/ICCggI+\n9alPsX79+qyukez+VpJ01QiDVNZbf/bt20dbWxs1NTVUVVVRVVXV6/vvfOc7/OhHP2Lnzp3s3buX\nQ4cOZVV/nx3i1q1bCfppfx988MGsLihJ0kA0NDRQXl4OwJQpUzh9+jQdHR0UFRVx7NgxRo8ezQ03\n3ADApz/9aRoaGpg6deqAr9NnIE6cODHL0iVJ16SYhkzT6TSlpaU9+8XFxbS3t1NUVER7ezvFxcW9\nvjt27FhW1+kzEJcvX97z92uvvcapU6cA6OzspKqqqtf3kiSFOZpUE4ZhLOe95KSazZs3c+DAAdra\n2igtLeXAgQOsXr06lmIkSckVU05RUlJCOp3u2T958iTjxo173+9OnDhBSUlJVte5ZH/7z3/+k6ef\nfpqpU6fys5/9jGeeeYYjR45kdTFJ0uCVCcOst/6UlZVRX18PQEtLCyUlJRQVFQFw44030tHRweuv\nv86FCxfYs2cPZWVlWdV/yQ6xu7ubM2fOEIYhb731FuPHj+fVV1/N6mKSpMErpgaRuXPnUlpaSkVF\nBUEQsGnTJurq6hg5ciSLFy/m29/+Ng899BAAS5cuZfLkyVldJwgvMRj77LPPcu7cOUaNGkVVVRWF\nhYXMnz+f7373u5d1gc7T6UsfpES6/3/m5bsExeTRs/6nd7AaNnRobOc+feZs1r8dPWJYhJVk55Id\n4rRp0/jYxz4GwG233cY777zDa6+9FnthkiTlUp+BeOzYMY4cOcL3v/99vva1r/V83t3dzebNm3nh\nhRdyUqAkKRnimv2ZK30G4pkzZ/jrX/9KOp3m2Wef7fk8lUqxbt26nBQnSUqOTLLzsO9AnDFjBjNm\nzGDJkiVMmjSJtrY2CgoKmDhxIoWFhbmsUZKUAAnPw0vfQzx06BD33XcfH/nIR+js7OT48eNs2LCB\n2267LRf1SZISYtB2iO966qmn2LVrF8OHDwego6ODe+65x0CUJPUyaO8hvqugoKAnDAGKioq47jrf\nGiVJ6i2T7wKu0CWTbfbs2dx7770sWLCAMAxpbGxkzpw5uahNkqSc6fPRbffffz8AGzZsYNWqVVy4\ncIFMJsPdd9/d80QASZLeFYbZb1eDPjvEd99uAXDzzTdz880356QgSVIyDdpJNUePHuV73/tenz/8\n+te/HktBkqRkGrSTaoYNG8a0adNyWYskKcEG7aSaD37wg3z+85/PZS2SpARLeIPYdyDOnDkzl3VI\nkhLuUu81vNr1Oct0w4YNuaxDkqS8coW9JCkSye4PDURJUkQG7bILSZIGIuG3EA1ESVI0MgkfNDUQ\nJUmRSHqH2OcsU0mSriV2iJKkSDipRpIkkj9kaiBKkiLhpBpJkrBDlCQJSP6zTA1ESVIkuhP+/ieX\nXUiShB2iJCkiDplKkgR0G4iSJNkhSpIEJH9SjYEoSYqEHaIkSST/HqLLLiRJwg5RkhQR33YhSRLQ\nnfBENBAlSZFwUo0kSUB3svPQQJQkRcMOUZIkkn8P0WUXkiRhhyhJiohDppIk4aQaSZIAO8RLyhSO\niPsSypNHz76a7xIUkweGzch3CYrJT8MjsZ07k/BJNXaIkqRIOGQqSRLJHzJ12YUkSdghSpIikvT3\nIRqIkqRI5HJSTVdXFxs3buT48eMUFBTwyCOPMGHChPc99sEHH6SwsJDq6up+z+mQqSQpEt1h9ttA\nPf/884waNYqdO3eydu1atm7d+r7H7d27l6NHj17WOQ1ESVIkMmGY9TZQDQ0NLF68GIBFixbx8ssv\nv+eYzs5OfvKTn7Bu3brLOqdDppKkSOTyHmI6naa4uBiAVCpFEAR0dnZSWFjYc8y2bdtYsWIFRUVF\nl3VOA1GSFIm43nZRW1tLbW1tr8+ampp67YcXhfGRI0dobm7my1/+Mo2NjZd1HQNRknRVW758OcuX\nL+/12caNG2lvb2fGjBl0dXURhmGv7vCPf/wjx48f5wtf+AIdHR28+eabbN++nTVr1vR5HQNRkhSJ\nXL4PsaysjN27d/PJT36SPXv2sHDhwl7f33XXXdx1110ANDY28utf/7rfMAQn1UiSItKdCbPeBmrp\n0qVkMhlWrFjBjh07eOihhwB44okn+Nvf/pZV/UF48cBrxM6dPRvn6ZVHYRDkuwTFxId7D15xPty7\nes+/sv7txlunRVhJdhwylSRFIpdDpnEwECVJkTAQJUki+YHopBpJkrBDlCRFJOkdooEoSYqEgShJ\nEgaiJEkAXDAQJUmyQ5QkCUh+ILrsQpIk7BAlSRHJ5QuC42AgSpIikfQhUwNRkhQJA1GSJAxESZIA\n6M5k8l3CFTEQJUmRSHqH6LILSZKwQ5QkRSTpHaKBKEmKhM8ylSQJO0RJkgADUZIkwECUJAlIfiC6\n7EKSJOwQJUkRSXqHaCBKkiIRGoiSJEHGQJQkCUJfECxJkkOmkiQByR8yddmFJElcZod45swZ0uk0\nAOPGjWP48OGxFiVJSp4w2e8H7j8Q9+/fT1VVFW+//TZjxowhDENOnjzJ9ddfz7e+9S2mT5+eqzol\nSVe5QT2pZsuWLVRVVTFlypRen7e0tPDwww+zY8eOWIuTJCVH0u8h9huIYRi+JwwBSktL6e7ujq0o\nSVLyDOpZprNnz2bt2rWUl5dTXFwMQDqdpr6+ngULFuSkQElSMgzqQPzGN77BSy+9RENDA6+88goA\nJSUl3HfffcyZMycnBUqSkiEzmO8hAsyfP5/58+fnohZJkvLGhfmSpEgM6iFTSZIul4EoSRKDfNmF\nJEmXa1AvzJck6XIN6ke3SZJ0uZI+ZOrbLiRJwg5RkhQRZ5lKkoSBKEkScA08uk2SpMthhyhJEgai\nJElAbpdddHV1sXHjRo4fP05BQQGPPPIIEyZM6HXMo48+SmNjI2EYUl5ezpo1a/o9p8suJEmJ8/zz\nzzNq1Ch27tzJ2rVr2bp1a6/vDx48SGNjI08//TQ7d+6krq6O9vb2fs9pIEqSIhGGYdbbQDU0NLB4\n8WIAFi1axMsvv9zr+5EjR3L+/Hk6Ozs5f/48qVSKYcOG9XtOh0wlSZHI5T3EdDpNcXExAKlUiiAI\n6OzspLCwEIAbbriBJUuWcOutt9Ld3c369espKirq95wGoiQpEnHdQ6ytraW2trbXZ01NTb32L+4y\njx07xu9+9zt+//vfc+HCBSoqKli6dCljx47t8zoGoiQpEmGmO5bzLl++nOXLl/f6bOPGjbS3tzNj\nxgy6uroIw7CnOwTYv38/s2fP7hkmnT59OgcPHuSWW27p8zreQ5QkRSLMdGe9DVRZWRm7d+8GYM+e\nPSxcuLDX9xMnTqS5uZlMJkNXVxcHDx58zyzUi9khSpIiEVeH+H6WLl3Kiy++yIoVKygsLKS6uhqA\nJ554gvnz5zNnzhzKysr44he/CMCyZcu48cYb+z1nEMb8RsdzZ8/GeXrlURgE+S5BMXlg2Ix8l6CY\n/DQ8Etu5J971VNa/Pfq/d0ZYSXbsECVJkQi7c9chxsFAlCRFIpdDpnEwECVJkTAQJUnCQJQkCTAQ\nJUkCkh+ILsyXJAk7RElSRDIJ7xANRElSJJI+ZGogSpIiYSBKkoRPqpEkCbBDlCQJSH4guuxCkiTs\nECVJEUl6h2ggSpIiEWYy+S7hihiIkqRI2CFKkoSBKEkS4KPbJEkCkr8w32UXkiRhhyhJioj3ECVJ\nwkCUJAkwECVJApIfiEEYhmG+i5AkKd+cZSpJEgaiJEmAgShJEmAgSpIEGIiSJAEGoiRJgOsQB+T1\n11/nc5/7HDNnzuz5bP/+/dx00028/fbbnDhxgmnTpgHw5JNPsmTJEj70oQ9RUFDQc/y9997LLbfc\nkvPa9f6qq6tpaWmhvb2ds2fPMnHiREaPHs3jjz/+nmMPHDjA8OHD+fCHP/y+56qtraWtrY2vfvWr\ncZetiGzZsoWmpiaCIKCyspJZs2bluyTlkYE4QJMnT+app556z+eNjY3s2LGDxx57rNfn27dvZ8SI\nEbkqTwO0ceNGAOrq6vjXv/7Fhg0b+jx29+7dfOITn+gzEJUs+/bto62tjZqaGlpbW6msrKSmpibf\nZSmPDETpfVRXV9PU1MSFCxdYtWoVU6dOpba2lhdeeIHi4mIOHTrEL3/5S1KpFNOnT2fz5s35LlkD\n1NDQQHl5OQBTpkzh9OnTdHR0UFRUlOfKlC8GonSRhoYGjhw5ws6dOzlz5gy33347zz33HIsWLeL2\n229n5syZ7N+/n5///OcUFRVRUVFBa2trvsvWAKXTaUpLS3v2i4uLaW9vNxCvYQbiAB0+fJg777yz\nZ3/RokWsW7euz+PXrFnT6x7i9u3bGTp0aKw16so0NzezYMECAEaMGMHkyZM5evRor2NGjx7Nl770\nJYIg4PDhw7z11lv5KFUR8imWMhAHqK97iH3xHmLyBEHQ6x/Hrq4uUqn/PyH7/PnzVFVVsWvXLsaO\nHcvq1avzUaauUElJCel0umf/5MmTjBs3Lo8VKd9cdiFd5KabbqKxsRGAjo4O3njjDSZOnEgqlaK7\nu5t33nmHwsJCxo4dyxtvvME//vEPurq68ly1BqqsrIz6+noAWlpaKCkpcbj0GmeHGLOLh0w/+9nP\ncscdd+SxIl3KwoUL+dOf/sTKlSu5cOECGzZsYOjQocybN4+HH36Y6upq5s2bx7Jly/joRz/K6tWr\nqaqq6jWUrqvf3LlzKS0tpaKigiAI2LRpU75LUp75+idJknDIVJIkwECUJAkwECVJAgxESZIAA1GS\nJMBAlCQJMBAlSQIMREmSAPh/e4E6vozxTkAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x396 with 2 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.heatmap(cov_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sRqBcqnK2iyX"
   },
   "source": [
    "As we can see, the numeric columns have a "
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "2_0_mayra_preprocessing-pca.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}