demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([[1,2],[1,2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1667633790"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(datetime.now().timestamp())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1\n",
       "0  1  2\n",
       "1  1  2"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1\n",
       "0  1  2\n",
       "1  1  2\n",
       "0  1  2\n",
       "1  1  2"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([df,df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sensor.utils.main_utils import read_yaml_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path=\"/home/avnish/iNeuron_Private_Intelligence_Limited/industry_ready_project/live_class/sensor-fault-detection/config/schema.yaml\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "schema_config = read_yaml_file(file_path=file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "164"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(schema_config['columns'])==len(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import ks_2samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "d1 = np.arange(100,200)\n",
    "d2 = np.arange(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112,\n",
       "       113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125,\n",
       "       126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138,\n",
       "       139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151,\n",
       "       152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164,\n",
       "       165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177,\n",
       "       178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190,\n",
       "       191, 192, 193, 194, 195, 196, 197, 198, 199])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112,\n",
       "       113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125,\n",
       "       126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138,\n",
       "       139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151,\n",
       "       152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164,\n",
       "       165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177,\n",
       "       178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190,\n",
       "       191, 192, 193, 194, 195, 196, 197, 198, 199])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = ks_2samp(d1,d2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(test.pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "traing_file_path = \"/home/avnish/iNeuron_Private_Intelligence_Limited/industry_ready_project/live_class/sensor-fault-detection/artifact/10_29_2022_12_29_11/data_ingestion/ingested/train.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_path = \"/home/avnish/iNeuron_Private_Intelligence_Limited/industry_ready_project/live_class/sensor-fault-detection/artifact/10_29_2022_12_29_11/data_ingestion/ingested/test.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(traing_file_path)\n",
    "test_df = pd.read_csv(test_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['class', 'aa_000', 'ab_000', 'ac_000', 'ad_000', 'ae_000', 'af_000',\n",
       "       'ag_000', 'ag_001', 'ag_002',\n",
       "       ...\n",
       "       'ee_002', 'ee_003', 'ee_004', 'ee_005', 'ee_006', 'ee_007', 'ee_008',\n",
       "       'ee_009', 'ef_000', 'eg_000'],\n",
       "      dtype='object', length=171)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['class', 'aa_000', 'ab_000', 'ac_000', 'ad_000', 'ae_000', 'af_000',\n",
       "       'ag_000', 'ag_001', 'ag_002',\n",
       "       ...\n",
       "       'ee_002', 'ee_003', 'ee_004', 'ee_005', 'ee_006', 'ee_007', 'ee_008',\n",
       "       'ee_009', 'ef_000', 'eg_000'],\n",
       "      dtype='object', length=171)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sensor.utils.main_utils import write_yaml_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def get_drift_report(base_df,current_df,threshold=0.05):\n",
    "    report ={}\n",
    "    for column in base_df.columns:\n",
    "        d1 = base_df[column]\n",
    "        d2  = current_df[column]\n",
    "        is_same_dist = ks_2samp(d1,d2)\n",
    "        if threshold<=is_same_dist.pvalue:\n",
    "            is_found=False\n",
    "        else:\n",
    "            is_found = True \n",
    "        report.update({column:{\n",
    "            \"p_value\":float(is_same_dist.pvalue),\n",
    "            \"drift_status\":is_found\n",
    "            \n",
    "            }})\n",
    "    return report\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "drift_report = get_drift_report(train_df,test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class': {'p_value': 1.0, 'drift_status': False},\n",
       " 'aa_000': {'p_value': 0.6183763526162667, 'drift_status': False},\n",
       " 'ab_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'ac_000': {'p_value': 0.9600616950774589, 'drift_status': False},\n",
       " 'ad_000': {'p_value': 0.8687879398544884, 'drift_status': False},\n",
       " 'ae_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'af_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'ag_000': {'p_value': 0.9999999999999267, 'drift_status': False},\n",
       " 'ag_001': {'p_value': 1.0, 'drift_status': False},\n",
       " 'ag_002': {'p_value': 0.9999999999999981, 'drift_status': False},\n",
       " 'ag_003': {'p_value': 0.9993680814508632, 'drift_status': False},\n",
       " 'ag_004': {'p_value': 0.5408095681164378, 'drift_status': False},\n",
       " 'ag_005': {'p_value': 0.06647425046922661, 'drift_status': False},\n",
       " 'ag_006': {'p_value': 0.3376001549603185, 'drift_status': False},\n",
       " 'ag_007': {'p_value': 0.6787464323185937, 'drift_status': False},\n",
       " 'ag_008': {'p_value': 0.7631175605685587, 'drift_status': False},\n",
       " 'ag_009': {'p_value': 0.804431987787424, 'drift_status': False},\n",
       " 'ah_000': {'p_value': 0.5873805748359822, 'drift_status': False},\n",
       " 'ai_000': {'p_value': 0.9999860478918007, 'drift_status': False},\n",
       " 'aj_000': {'p_value': 0.7758966606201321, 'drift_status': False},\n",
       " 'ak_000': {'p_value': 0.9999969125429781, 'drift_status': False},\n",
       " 'al_000': {'p_value': 0.9322718774835487, 'drift_status': False},\n",
       " 'am_0': {'p_value': 0.974083553086165, 'drift_status': False},\n",
       " 'an_000': {'p_value': 0.4498061707181127, 'drift_status': False},\n",
       " 'ao_000': {'p_value': 0.426657727779545, 'drift_status': False},\n",
       " 'ap_000': {'p_value': 0.5304841586431874, 'drift_status': False},\n",
       " 'aq_000': {'p_value': 0.6360467243395488, 'drift_status': False},\n",
       " 'ar_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'as_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'at_000': {'p_value': 0.9998638520347088, 'drift_status': False},\n",
       " 'au_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'av_000': {'p_value': 0.5275150499416327, 'drift_status': False},\n",
       " 'ax_000': {'p_value': 0.7330511743762196, 'drift_status': False},\n",
       " 'ay_000': {'p_value': 0.9999999999998291, 'drift_status': False},\n",
       " 'ay_001': {'p_value': 0.9999989513893514, 'drift_status': False},\n",
       " 'ay_002': {'p_value': 0.9999947379267049, 'drift_status': False},\n",
       " 'ay_003': {'p_value': 0.9999744916658864, 'drift_status': False},\n",
       " 'ay_004': {'p_value': 0.9907241954388551, 'drift_status': False},\n",
       " 'ay_005': {'p_value': 0.4356673902896172, 'drift_status': False},\n",
       " 'ay_006': {'p_value': 0.7780967026085766, 'drift_status': False},\n",
       " 'ay_007': {'p_value': 0.46468977086554186, 'drift_status': False},\n",
       " 'ay_008': {'p_value': 0.34776688822288004, 'drift_status': False},\n",
       " 'ay_009': {'p_value': 1.0, 'drift_status': False},\n",
       " 'az_000': {'p_value': 0.93712692903771, 'drift_status': False},\n",
       " 'az_001': {'p_value': 0.2513284735040556, 'drift_status': False},\n",
       " 'az_002': {'p_value': 0.2819339260390529, 'drift_status': False},\n",
       " 'az_003': {'p_value': 0.781851140788774, 'drift_status': False},\n",
       " 'az_004': {'p_value': 0.8351727476513281, 'drift_status': False},\n",
       " 'az_005': {'p_value': 0.22531239170819017, 'drift_status': False},\n",
       " 'az_006': {'p_value': 0.679408697926631, 'drift_status': False},\n",
       " 'az_007': {'p_value': 0.9999999999757986, 'drift_status': False},\n",
       " 'az_008': {'p_value': 0.9999999991111189, 'drift_status': False},\n",
       " 'az_009': {'p_value': 0.9999999999996254, 'drift_status': False},\n",
       " 'ba_000': {'p_value': 0.505963582151906, 'drift_status': False},\n",
       " 'ba_001': {'p_value': 0.7716099811823537, 'drift_status': False},\n",
       " 'ba_002': {'p_value': 0.46134064288746657, 'drift_status': False},\n",
       " 'ba_003': {'p_value': 0.7541723006089878, 'drift_status': False},\n",
       " 'ba_004': {'p_value': 0.6403676320699525, 'drift_status': False},\n",
       " 'ba_005': {'p_value': 0.30870130256437067, 'drift_status': False},\n",
       " 'ba_006': {'p_value': 0.08827791620349412, 'drift_status': False},\n",
       " 'ba_007': {'p_value': 0.5271459735616915, 'drift_status': False},\n",
       " 'ba_008': {'p_value': 0.3539872050311559, 'drift_status': False},\n",
       " 'ba_009': {'p_value': 0.052538677308316406, 'drift_status': False},\n",
       " 'bb_000': {'p_value': 0.3214505258275151, 'drift_status': False},\n",
       " 'bc_000': {'p_value': 0.16709059687016925, 'drift_status': False},\n",
       " 'bd_000': {'p_value': 0.7893999842851589, 'drift_status': False},\n",
       " 'be_000': {'p_value': 0.7106994141634022, 'drift_status': False},\n",
       " 'bf_000': {'p_value': 0.8502582832776324, 'drift_status': False},\n",
       " 'bg_000': {'p_value': 0.5961458133147896, 'drift_status': False},\n",
       " 'bh_000': {'p_value': 0.7836222151356347, 'drift_status': False},\n",
       " 'bi_000': {'p_value': 0.21992860518539004, 'drift_status': False},\n",
       " 'bj_000': {'p_value': 0.8972272722192669, 'drift_status': False},\n",
       " 'bk_000': {'p_value': 0.08372493359953093, 'drift_status': False},\n",
       " 'bl_000': {'p_value': 0.09090494365554824, 'drift_status': False},\n",
       " 'bm_000': {'p_value': 0.9700914623746663, 'drift_status': False},\n",
       " 'bn_000': {'p_value': 0.9139841139900792, 'drift_status': False},\n",
       " 'bo_000': {'p_value': 0.6750364657781234, 'drift_status': False},\n",
       " 'bp_000': {'p_value': 0.3994146983761049, 'drift_status': False},\n",
       " 'bq_000': {'p_value': 0.4725451863625867, 'drift_status': False},\n",
       " 'br_000': {'p_value': 0.8147350803248641, 'drift_status': False},\n",
       " 'bs_000': {'p_value': 0.23186616801914084, 'drift_status': False},\n",
       " 'bt_000': {'p_value': 0.6626880180556431, 'drift_status': False},\n",
       " 'bu_000': {'p_value': 0.32464791186429487, 'drift_status': False},\n",
       " 'bv_000': {'p_value': 0.32464791186429487, 'drift_status': False},\n",
       " 'bx_000': {'p_value': 0.43791217014477113, 'drift_status': False},\n",
       " 'by_000': {'p_value': 0.684498440154508, 'drift_status': False},\n",
       " 'bz_000': {'p_value': 0.7632969077344696, 'drift_status': False},\n",
       " 'ca_000': {'p_value': 0.15094223729690526, 'drift_status': False},\n",
       " 'cb_000': {'p_value': 0.042210687009804126, 'drift_status': True},\n",
       " 'cc_000': {'p_value': 0.4301659808103575, 'drift_status': False},\n",
       " 'cd_000': {'p_value': 0.9999999999980729, 'drift_status': False},\n",
       " 'ce_000': {'p_value': 0.8388112392806355, 'drift_status': False},\n",
       " 'cf_000': {'p_value': 0.9671838703145905, 'drift_status': False},\n",
       " 'cg_000': {'p_value': 0.8653107308595918, 'drift_status': False},\n",
       " 'ch_000': {'p_value': 0.99980194001844, 'drift_status': False},\n",
       " 'ci_000': {'p_value': 0.6758838020594136, 'drift_status': False},\n",
       " 'cj_000': {'p_value': 0.4811288352458275, 'drift_status': False},\n",
       " 'ck_000': {'p_value': 0.5235609750769215, 'drift_status': False},\n",
       " 'cl_000': {'p_value': 0.8085209527191395, 'drift_status': False},\n",
       " 'cm_000': {'p_value': 0.9867275474728535, 'drift_status': False},\n",
       " 'cn_000': {'p_value': 0.9999999956750366, 'drift_status': False},\n",
       " 'cn_001': {'p_value': 0.9996740408229148, 'drift_status': False},\n",
       " 'cn_002': {'p_value': 0.6737634026178645, 'drift_status': False},\n",
       " 'cn_003': {'p_value': 0.10290327274728384, 'drift_status': False},\n",
       " 'cn_004': {'p_value': 0.3746541751470259, 'drift_status': False},\n",
       " 'cn_005': {'p_value': 0.3440659336851537, 'drift_status': False},\n",
       " 'cn_006': {'p_value': 0.476316680792933, 'drift_status': False},\n",
       " 'cn_007': {'p_value': 0.42295511214072246, 'drift_status': False},\n",
       " 'cn_008': {'p_value': 0.5480677623785941, 'drift_status': False},\n",
       " 'cn_009': {'p_value': 0.5437472581497079, 'drift_status': False},\n",
       " 'co_000': {'p_value': 0.5051808379759775, 'drift_status': False},\n",
       " 'cp_000': {'p_value': 0.08149124214110492, 'drift_status': False},\n",
       " 'cq_000': {'p_value': 0.32464791186429487, 'drift_status': False},\n",
       " 'cr_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'cs_000': {'p_value': 0.8198087805906651, 'drift_status': False},\n",
       " 'cs_001': {'p_value': 0.5524475213245992, 'drift_status': False},\n",
       " 'cs_002': {'p_value': 0.8679812700588057, 'drift_status': False},\n",
       " 'cs_003': {'p_value': 0.5614658687392585, 'drift_status': False},\n",
       " 'cs_004': {'p_value': 0.7192791892130774, 'drift_status': False},\n",
       " 'cs_005': {'p_value': 0.775718616617564, 'drift_status': False},\n",
       " 'cs_006': {'p_value': 0.10289708512850015, 'drift_status': False},\n",
       " 'cs_007': {'p_value': 0.39651741504102966, 'drift_status': False},\n",
       " 'cs_008': {'p_value': 0.542979534882713, 'drift_status': False},\n",
       " 'cs_009': {'p_value': 0.9999997874624981, 'drift_status': False},\n",
       " 'ct_000': {'p_value': 0.9320504180107787, 'drift_status': False},\n",
       " 'cu_000': {'p_value': 0.9773584942619228, 'drift_status': False},\n",
       " 'cv_000': {'p_value': 0.9389388441670303, 'drift_status': False},\n",
       " 'cx_000': {'p_value': 0.5812432784107548, 'drift_status': False},\n",
       " 'cy_000': {'p_value': 0.9686814212360937, 'drift_status': False},\n",
       " 'cz_000': {'p_value': 0.841726906766219, 'drift_status': False},\n",
       " 'da_000': {'p_value': 0.9999999995645833, 'drift_status': False},\n",
       " 'db_000': {'p_value': 0.9999999706726543, 'drift_status': False},\n",
       " 'dc_000': {'p_value': 0.9695284391141507, 'drift_status': False},\n",
       " 'dd_000': {'p_value': 0.9441005127059542, 'drift_status': False},\n",
       " 'de_000': {'p_value': 0.33944480372285213, 'drift_status': False},\n",
       " 'df_000': {'p_value': 0.9999992401496168, 'drift_status': False},\n",
       " 'dg_000': {'p_value': 0.9999986339398381, 'drift_status': False},\n",
       " 'dh_000': {'p_value': 0.2784362981652184, 'drift_status': False},\n",
       " 'di_000': {'p_value': 0.385599540085349, 'drift_status': False},\n",
       " 'dj_000': {'p_value': 0.9997566328948778, 'drift_status': False},\n",
       " 'dk_000': {'p_value': 0.9990804177874175, 'drift_status': False},\n",
       " 'dl_000': {'p_value': 0.9995700058210619, 'drift_status': False},\n",
       " 'dm_000': {'p_value': 0.9989656852995394, 'drift_status': False},\n",
       " 'dn_000': {'p_value': 0.8194359265095817, 'drift_status': False},\n",
       " 'do_000': {'p_value': 0.7478289877167011, 'drift_status': False},\n",
       " 'dp_000': {'p_value': 0.6050610107628747, 'drift_status': False},\n",
       " 'dq_000': {'p_value': 0.9933208645471703, 'drift_status': False},\n",
       " 'dr_000': {'p_value': 0.999804825412129, 'drift_status': False},\n",
       " 'ds_000': {'p_value': 0.7741110980228215, 'drift_status': False},\n",
       " 'dt_000': {'p_value': 0.7185805935958807, 'drift_status': False},\n",
       " 'du_000': {'p_value': 0.43688501130098245, 'drift_status': False},\n",
       " 'dv_000': {'p_value': 0.38859639606010477, 'drift_status': False},\n",
       " 'dx_000': {'p_value': 0.9965381374319506, 'drift_status': False},\n",
       " 'dy_000': {'p_value': 0.9998917847782711, 'drift_status': False},\n",
       " 'dz_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'ea_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'eb_000': {'p_value': 0.4010082399892698, 'drift_status': False},\n",
       " 'ec_00': {'p_value': 0.6301716088888111, 'drift_status': False},\n",
       " 'ed_000': {'p_value': 0.8343044632899759, 'drift_status': False},\n",
       " 'ee_000': {'p_value': 0.4184675893197981, 'drift_status': False},\n",
       " 'ee_001': {'p_value': 0.3891875089790934, 'drift_status': False},\n",
       " 'ee_002': {'p_value': 0.6214766406673984, 'drift_status': False},\n",
       " 'ee_003': {'p_value': 0.4653306287450034, 'drift_status': False},\n",
       " 'ee_004': {'p_value': 0.6975435499061089, 'drift_status': False},\n",
       " 'ee_005': {'p_value': 0.8595351668204118, 'drift_status': False},\n",
       " 'ee_006': {'p_value': 0.40066811349668163, 'drift_status': False},\n",
       " 'ee_007': {'p_value': 0.38611935948416676, 'drift_status': False},\n",
       " 'ee_008': {'p_value': 0.4301078003636625, 'drift_status': False},\n",
       " 'ee_009': {'p_value': 0.47504040509081125, 'drift_status': False},\n",
       " 'ef_000': {'p_value': 1.0, 'drift_status': False},\n",
       " 'eg_000': {'p_value': 1.0, 'drift_status': False}}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drift_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_yaml_file(file_path=os.path.join(os.getcwd(),\"report.yaml\"),content=drift_report,replace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KstestResult(statistic=0.0005161971383997876, pvalue=1.0)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d1 = train_df['class']\n",
    "d2  = test_df['class']\n",
    "ks_2samp(d1,d2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KstestResult(statistic=0.009896540089462558, pvalue=0.6183763526162667)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sensor.ml.model.estimator import TargetValueMapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'neg': 0, 'pos': 1}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".to_dict()TargetValueMapping()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'neg', 1: 'pos'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TargetValueMapping().reverse_mapping()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}