day65-random-forest/code-example-random-forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "20f54af6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "d32397f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('heart.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "fc33a557",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "75b9725a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(303, 14)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "17bd4c40",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.iloc[:,0:-1]\n",
    "y = df.iloc[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "16f4079f",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "938f9b7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(242, 13)\n",
      "(61, 13)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "c14abae7",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "rf = RandomForestClassifier()\n",
    "gb = GradientBoostingClassifier()\n",
    "svc = SVC()\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "d5a8716a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.fit(X_train,y_train)\n",
    "y_pred = rf.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "1fab7ecd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7704918032786885"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb.fit(X_train,y_train)\n",
    "y_pred = gb.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "5f4d1a14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7049180327868853"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svc.fit(X_train,y_train)\n",
    "y_pred = svc.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "df55e7ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\91842\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8852459016393442"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.fit(X_train,y_train)\n",
    "y_pred = lr.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "ea7a757f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9016393442622951"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf = RandomForestClassifier(max_samples=0.75,random_state=42)\n",
    "rf.fit(X_train,y_train)\n",
    "y_pred = rf.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "2c7d7317",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8412903225806451"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "np.mean(cross_val_score(RandomForestClassifier(max_samples=0.75),X,y,cv=10,scoring='accuracy'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8944923f",
   "metadata": {},
   "source": [
    "### GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "54684e72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of trees in random forest\n",
    "n_estimators = [20,60,100,120]\n",
    "\n",
    "# Number of features to consider at every split\n",
    "max_features = [0.2,0.6,1.0]\n",
    "\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [2,8,None]\n",
    "\n",
    "# Number of samples\n",
    "max_samples = [0.5,0.75,1.0]\n",
    "\n",
    "# 108 diff random forest train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "2698a056",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_estimators': [20, 60, 100, 120], 'max_features': [0.2, 0.6, 1.0], 'max_depth': [2, 8, None], 'max_samples': [0.5, 0.75, 1.0]}\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "              'max_samples':max_samples\n",
    "             }\n",
    "print(param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "a4ee39e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "6ee31025",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "rf_grid = GridSearchCV(estimator = rf, \n",
    "                       param_grid = param_grid, \n",
    "                       cv = 5, \n",
    "                       verbose=2, \n",
    "                       n_jobs = -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "c82c0fb9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 108 candidates, totalling 540 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\91842\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:918: UserWarning: One or more of the test scores are non-finite: [0.78894558 0.80552721 0.82627551 0.81802721 0.82219388 0.80552721\n",
      " 0.81802721 0.81794218        nan        nan        nan        nan\n",
      " 0.77236395 0.81394558 0.80986395 0.80977891 0.79761905 0.80161565\n",
      " 0.80994898 0.80986395        nan        nan        nan        nan\n",
      " 0.78920068 0.80170068 0.80994898 0.80170068 0.77695578 0.80153061\n",
      " 0.79336735 0.80578231        nan        nan        nan        nan\n",
      " 0.80161565 0.80960884 0.79753401 0.82202381 0.78903061 0.82636054\n",
      " 0.82219388 0.83035714        nan        nan        nan        nan\n",
      " 0.79753401 0.79753401 0.80569728 0.80986395 0.78103741 0.78095238\n",
      " 0.80153061 0.80178571        nan        nan        nan        nan\n",
      " 0.80170068 0.80144558 0.79753401 0.80969388 0.80178571 0.78920068\n",
      " 0.80586735 0.80578231        nan        nan        nan        nan\n",
      " 0.81802721 0.82219388 0.80569728 0.82236395 0.83069728 0.80986395\n",
      " 0.79744898 0.81394558        nan        nan        nan        nan\n",
      " 0.78095238 0.81411565 0.79345238 0.80578231 0.79761905 0.80170068\n",
      " 0.77270408 0.80986395        nan        nan        nan        nan\n",
      " 0.80586735 0.81386054 0.79753401 0.78095238 0.82627551 0.80586735\n",
      " 0.80153061 0.78920068        nan        nan        nan        nan]\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, estimator=RandomForestClassifier(), n_jobs=-1,\n",
       "             param_grid={'max_depth': [2, 8, None],\n",
       "                         'max_features': [0.2, 0.6, 1.0],\n",
       "                         'max_samples': [0.5, 0.75, 1.0],\n",
       "                         'n_estimators': [20, 60, 100, 120]},\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "90921870",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': None,\n",
       " 'max_features': 0.2,\n",
       " 'max_samples': 0.75,\n",
       " 'n_estimators': 20}"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "62eb5ec7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8306972789115645"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "087ce49e",
   "metadata": {},
   "source": [
    "### RandomSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "f79a9fd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of trees in random forest\n",
    "n_estimators = [20,60,100,120]\n",
    "\n",
    "# Number of features to consider at every split\n",
    "max_features = [0.2,0.6,1.0]\n",
    "\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [2,8,None]\n",
    "\n",
    "# Number of samples\n",
    "max_samples = [0.5,0.75,1.0]\n",
    "\n",
    "# Bootstrap samples\n",
    "bootstrap = [True,False]\n",
    "\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5]\n",
    "\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "15c54414",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_estimators': [20, 60, 100, 120], 'max_features': [0.2, 0.6, 1.0], 'max_depth': [2, 8, None], 'max_samples': [0.5, 0.75, 1.0], 'bootstrap': [True, False], 'min_samples_split': [2, 5], 'min_samples_leaf': [1, 2]}\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "              'max_samples':max_samples,\n",
    "              'bootstrap':bootstrap,\n",
    "              'min_samples_split':min_samples_split,\n",
    "              'min_samples_leaf':min_samples_leaf\n",
    "             }\n",
    "print(param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "bbd6bb8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "rf_grid = RandomizedSearchCV(estimator = rf, \n",
    "                       param_distributions = param_grid, \n",
    "                       cv = 5, \n",
    "                       verbose=2, \n",
    "                       n_jobs = -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "3ba35b1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\91842\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:918: UserWarning: One or more of the test scores are non-finite: [0.81785714 0.81411565 0.76454082 0.80977891 0.79328231        nan\n",
      " 0.81377551 0.80986395 0.80170068        nan]\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, estimator=RandomForestClassifier(), n_jobs=-1,\n",
       "                   param_distributions={'bootstrap': [True, False],\n",
       "                                        'max_depth': [2, 8, None],\n",
       "                                        'max_features': [0.2, 0.6, 1.0],\n",
       "                                        'max_samples': [0.5, 0.75, 1.0],\n",
       "                                        'min_samples_leaf': [1, 2],\n",
       "                                        'min_samples_split': [2, 5],\n",
       "                                        'n_estimators': [20, 60, 100, 120]},\n",
       "                   verbose=2)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "1d294b4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 120,\n",
       " 'min_samples_split': 2,\n",
       " 'min_samples_leaf': 2,\n",
       " 'max_samples': 0.75,\n",
       " 'max_features': 0.2,\n",
       " 'max_depth': None,\n",
       " 'bootstrap': True}"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "f1522007",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8178571428571428"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d91b9a9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}