day52-types-of-gradient-descent/mini-batch-gradient-descent-from-scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e1d1b3f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_diabetes\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "84500f46",
   "metadata": {},
   "outputs": [],
   "source": [
    "X,y = load_diabetes(return_X_y=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5ef471cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(442, 10)\n",
      "(442,)\n"
     ]
    }
   ],
   "source": [
    "print(X.shape)\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "01ebedbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2,random_state=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "3ab37e5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg = LinearRegression()\n",
    "reg.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "b0460e00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  -9.16088483 -205.46225988  516.68462383  340.62734108 -895.54360867\n",
      "  561.21453306  153.88478595  126.73431596  861.12139955   52.41982836]\n",
      "151.88334520854633\n"
     ]
    }
   ],
   "source": [
    "print(reg.coef_)\n",
    "print(reg.intercept_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "5c4104dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4399387660024645"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = reg.predict(X_test)\n",
    "r2_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "4d92f6d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "class MBGDRegressor:\n",
    "    \n",
    "    def __init__(self,batch_size,learning_rate=0.01,epochs=100):\n",
    "        \n",
    "        self.coef_ = None\n",
    "        self.intercept_ = None\n",
    "        self.lr = learning_rate\n",
    "        self.epochs = epochs\n",
    "        self.batch_size = batch_size\n",
    "        \n",
    "    def fit(self,X_train,y_train):\n",
    "        # init your coefs\n",
    "        self.intercept_ = 0\n",
    "        self.coef_ = np.ones(X_train.shape[1])\n",
    "        \n",
    "        for i in range(self.epochs):\n",
    "            \n",
    "            for j in range(int(X_train.shape[0]/self.batch_size)):\n",
    "                \n",
    "                idx = random.sample(range(X_train.shape[0]),self.batch_size)\n",
    "                \n",
    "                y_hat = np.dot(X_train[idx],self.coef_) + self.intercept_\n",
    "                #print(\"Shape of y_hat\",y_hat.shape)\n",
    "                intercept_der = -2 * np.mean(y_train[idx] - y_hat)\n",
    "                self.intercept_ = self.intercept_ - (self.lr * intercept_der)\n",
    "\n",
    "                coef_der = -2 * np.dot((y_train[idx] - y_hat),X_train[idx])\n",
    "                self.coef_ = self.coef_ - (self.lr * coef_der)\n",
    "        \n",
    "        print(self.intercept_,self.coef_)\n",
    "    \n",
    "    def predict(self,X_test):\n",
    "        return np.dot(X_test,self.coef_) + self.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "99078906",
   "metadata": {},
   "outputs": [],
   "source": [
    "mbr = MBGDRegressor(batch_size=int(X_train.shape[0]/50),learning_rate=0.01,epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "dc4e8574",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "154.83742294704174 [  38.40848324 -142.67633481  457.28746444  303.60926403  -17.99961807\n",
      "  -85.81943788 -192.04933133  116.18528414  407.24722272  105.8082595 ]\n"
     ]
    }
   ],
   "source": [
    "mbr.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "d32d9fef",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = mbr.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "905a915d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4518861449568681"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r2_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "a48ecad9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "cc4d285f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd = SGDRegressor(learning_rate='constant',eta0=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "23b8a8be",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 35\n",
    "\n",
    "for i in range(100):\n",
    "    \n",
    "    idx = random.sample(range(X_train.shape[0]),batch_size)\n",
    "    sgd.partial_fit(X_train[idx],y_train[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "9827c50a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  49.19545821,  -67.84533938,  338.57819421,  247.97315609,\n",
       "         25.30849249,  -24.71685159, -155.45845777,  116.19331239,\n",
       "        312.91250811,  133.36595993])"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "bf5ae439",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([148.61489911])"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "df9c2619",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = sgd.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "ab5ea374",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4271789125617129"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r2_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b8d5010",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d213087b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}