GRANDE_minimal_example_with_comparison_BINARY.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "19407948-2e33-4576-ac7a-1c76819b421a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:56:56.250625Z",
     "iopub.status.busy": "2024-03-15T11:56:56.250500Z",
     "iopub.status.idle": "2024-03-15T11:56:56.253023Z",
     "shell.execute_reply": "2024-03-15T11:56:56.252707Z",
     "shell.execute_reply.started": "2024-03-15T11:56:56.250606Z"
    }
   },
   "outputs": [],
   "source": [
    "!pip install catboost\n",
    "!pip install xgboost\n",
    "!pip install openml\n",
    "!pip install numba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bf584ce8-8849-4a53-8ce9-2de8f9dd752d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:56:56.253578Z",
     "iopub.status.busy": "2024-03-15T11:56:56.253464Z",
     "iopub.status.idle": "2024-03-15T11:56:56.265905Z",
     "shell.execute_reply": "2024-03-15T11:56:56.265512Z",
     "shell.execute_reply.started": "2024-03-15T11:56:56.253566Z"
    }
   },
   "outputs": [],
   "source": [
    "#specify GPU to use\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "52b07238-f972-45df-86ce-a11cc11d72e3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:56:56.266511Z",
     "iopub.status.busy": "2024-03-15T11:56:56.266392Z",
     "iopub.status.idle": "2024-03-15T11:56:57.956110Z",
     "shell.execute_reply": "2024-03-15T11:56:57.955562Z",
     "shell.execute_reply.started": "2024-03-15T11:56:56.266499Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size: 5361\n",
      "Validation set size: 1341\n",
      "Test set size: 1676\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>has_null</th>\n",
       "      <th>wave</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>age_o</th>\n",
       "      <th>d_age</th>\n",
       "      <th>d_d_age</th>\n",
       "      <th>race</th>\n",
       "      <th>race_o</th>\n",
       "      <th>samerace</th>\n",
       "      <th>...</th>\n",
       "      <th>expected_num_interested_in_me</th>\n",
       "      <th>expected_num_matches</th>\n",
       "      <th>d_expected_happy_with_sd_people</th>\n",
       "      <th>d_expected_num_interested_in_me</th>\n",
       "      <th>d_expected_num_matches</th>\n",
       "      <th>like</th>\n",
       "      <th>guess_prob_liked</th>\n",
       "      <th>d_like</th>\n",
       "      <th>d_guess_prob_liked</th>\n",
       "      <th>met</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>813</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>23.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>European/Caucasian-American</td>\n",
       "      <td>European/Caucasian-American</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>[0-4]</td>\n",
       "      <td>[10-20]</td>\n",
       "      <td>[3-5]</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>[0-5]</td>\n",
       "      <td>[7-10]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1677</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>female</td>\n",
       "      <td>20.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Black/African American</td>\n",
       "      <td>Asian/Pacific Islander/Asian-American</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>[5-6]</td>\n",
       "      <td>[0-3]</td>\n",
       "      <td>[3-5]</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>[6-8]</td>\n",
       "      <td>[7-10]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2915</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>female</td>\n",
       "      <td>28.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[0-1]</td>\n",
       "      <td>Black/African American</td>\n",
       "      <td>Asian/Pacific Islander/Asian-American</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[7-10]</td>\n",
       "      <td>[0-3]</td>\n",
       "      <td>[0-2]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[0-5]</td>\n",
       "      <td>[0-4]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5972</th>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>female</td>\n",
       "      <td>28.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>4</td>\n",
       "      <td>[4-6]</td>\n",
       "      <td>European/Caucasian-American</td>\n",
       "      <td>European/Caucasian-American</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[0-4]</td>\n",
       "      <td>[0-3]</td>\n",
       "      <td>[0-2]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>[0-5]</td>\n",
       "      <td>[5-6]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5443</th>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>male</td>\n",
       "      <td>23.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[0-1]</td>\n",
       "      <td>Asian/Pacific Islander/Asian-American</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>[7-10]</td>\n",
       "      <td>[0-3]</td>\n",
       "      <td>[3-5]</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>[0-5]</td>\n",
       "      <td>[0-4]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 120 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     has_null  wave  gender   age  age_o  d_age d_d_age  \\\n",
       "813         0     3  female  23.0   21.0      2   [2-3]   \n",
       "1677        1     5  female  20.0   18.0      2   [2-3]   \n",
       "2915        1     9  female  28.0   27.0      1   [0-1]   \n",
       "5972        1    15  female  28.0   24.0      4   [4-6]   \n",
       "5443        1    14    male  23.0   24.0      1   [0-1]   \n",
       "\n",
       "                                       race  \\\n",
       "813             European/Caucasian-American   \n",
       "1677                 Black/African American   \n",
       "2915                 Black/African American   \n",
       "5972            European/Caucasian-American   \n",
       "5443  Asian/Pacific Islander/Asian-American   \n",
       "\n",
       "                                     race_o samerace  ...  \\\n",
       "813             European/Caucasian-American        1  ...   \n",
       "1677  Asian/Pacific Islander/Asian-American        0  ...   \n",
       "2915  Asian/Pacific Islander/Asian-American        0  ...   \n",
       "5972            European/Caucasian-American        1  ...   \n",
       "5443                                  Other        0  ...   \n",
       "\n",
       "      expected_num_interested_in_me  expected_num_matches  \\\n",
       "813                            15.0                   3.0   \n",
       "1677                            NaN                   3.0   \n",
       "2915                            NaN                   0.0   \n",
       "5972                            NaN                   1.0   \n",
       "5443                            NaN                   4.0   \n",
       "\n",
       "     d_expected_happy_with_sd_people d_expected_num_interested_in_me  \\\n",
       "813                            [0-4]                         [10-20]   \n",
       "1677                           [5-6]                           [0-3]   \n",
       "2915                          [7-10]                           [0-3]   \n",
       "5972                           [0-4]                           [0-3]   \n",
       "5443                          [7-10]                           [0-3]   \n",
       "\n",
       "     d_expected_num_matches  like  guess_prob_liked  d_like  \\\n",
       "813                   [3-5]   5.0              10.0   [0-5]   \n",
       "1677                  [3-5]   7.0               7.0   [6-8]   \n",
       "2915                  [0-2]   NaN               NaN   [0-5]   \n",
       "5972                  [0-2]   3.0               5.0   [0-5]   \n",
       "5443                  [3-5]   5.0               4.0   [0-5]   \n",
       "\n",
       "      d_guess_prob_liked  met  \n",
       "813               [7-10]  0.0  \n",
       "1677              [7-10]  0.0  \n",
       "2915               [0-4]  NaN  \n",
       "5972               [5-6]  0.0  \n",
       "5443               [0-4]  0.0  \n",
       "\n",
       "[5 rows x 120 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import openml\n",
    "import category_encoders as ce\n",
    "import numpy as np\n",
    "import sklearn\n",
    "\n",
    "# Load SpeedDating dataset\n",
    "dataset = openml.datasets.get_dataset(40536, download_data=True, download_qualities=True, download_features_meta_data=True)\n",
    "X, y, categorical_indicator, attribute_names = dataset.get_data(target=dataset.default_target_attribute)\n",
    "categorical_feature_indices = [idx for idx, idx_bool in enumerate(categorical_indicator) if idx_bool]\n",
    "\n",
    "X_temp, X_test, y_temp, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_temp, y_temp, test_size=0.2, random_state=42)\n",
    "\n",
    "print(\"Training set size:\", len(X_train))\n",
    "print(\"Validation set size:\", len(X_valid))\n",
    "print(\"Test set size:\", len(X_test))\n",
    "\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bf0a93c2-e9a4-4fa5-a7f6-44689dd07d37",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:56:57.957002Z",
     "iopub.status.busy": "2024-03-15T11:56:57.956779Z",
     "iopub.status.idle": "2024-03-15T11:57:30.600318Z",
     "shell.execute_reply": "2024-03-15T11:57:30.599632Z",
     "shell.execute_reply.started": "2024-03-15T11:56:57.956987Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-15 12:56:58.283456: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-03-15 12:56:59.742100: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "2024-03-15 12:57:05.015127: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:47] Overriding orig_value setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
      "2024-03-15 12:57:05.015602: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1928] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 34057 MB memory:  -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:61:00.0, compute capability: 8.6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1710503828.442555 1478785 service.cc:145] XLA service 0x7fedac5cfa40 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "I0000 00:00:1710503828.442605 1478785 service.cc:153]   StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n",
      "2024-03-15 12:57:08.510371: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-03-15 12:57:08.817572: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:465] Loaded cuDNN version 8902\n",
      "I0000 00:00:1710503829.714629 1478785 device_compiler.h:188] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 33ms/step - loss: 0.6419 - val_loss: 0.5219\n",
      "Epoch 2/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 5ms/step - loss: 0.4886 - val_loss: 0.4299\n",
      "Epoch 3/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.4023 - val_loss: 0.3713\n",
      "Epoch 4/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 4ms/step - loss: 0.3420 - val_loss: 0.3483\n",
      "Epoch 5/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.3152 - val_loss: 0.3424\n",
      "Epoch 6/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.3036 - val_loss: 0.3412\n",
      "Epoch 7/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2978 - val_loss: 0.3401\n",
      "Epoch 8/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 5ms/step - loss: 0.2937 - val_loss: 0.3404\n",
      "Epoch 9/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 4ms/step - loss: 0.2902 - val_loss: 0.3409\n",
      "Epoch 10/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 4ms/step - loss: 0.2877 - val_loss: 0.3423\n",
      "Epoch 11/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2854 - val_loss: 0.3428\n",
      "Epoch 12/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2827 - val_loss: 0.3430\n",
      "Epoch 13/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2805 - val_loss: 0.3434\n",
      "Epoch 14/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 5ms/step - loss: 0.2780 - val_loss: 0.3431\n",
      "Epoch 15/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2763 - val_loss: 0.3444\n",
      "Epoch 16/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2739 - val_loss: 0.3444\n",
      "Epoch 17/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2714 - val_loss: 0.3471\n",
      "Epoch 18/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2695 - val_loss: 0.3462\n",
      "Epoch 19/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 5ms/step - loss: 0.2669 - val_loss: 0.3461\n",
      "Epoch 20/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2652 - val_loss: 0.3474\n",
      "Epoch 21/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2613 - val_loss: 0.3498\n",
      "Epoch 22/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2590 - val_loss: 0.3504\n",
      "Epoch 23/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2562 - val_loss: 0.3520\n",
      "Epoch 24/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2536 - val_loss: 0.3519\n",
      "Epoch 25/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 5ms/step - loss: 0.2512 - val_loss: 0.3539\n",
      "Epoch 26/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 4ms/step - loss: 0.2476 - val_loss: 0.3553\n",
      "Epoch 27/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 4ms/step - loss: 0.2447 - val_loss: 0.3552\n",
      "Epoch 28/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2409 - val_loss: 0.3566\n",
      "Epoch 29/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 4ms/step - loss: 0.2378 - val_loss: 0.3568\n",
      "Epoch 30/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2337 - val_loss: 0.3587\n",
      "Epoch 31/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 5ms/step - loss: 0.2282 - val_loss: 0.3590\n",
      "Epoch 32/1000\n",
      "\u001b[1m84/84\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - loss: 0.2257 - val_loss: 0.3622\n"
     ]
    }
   ],
   "source": [
    "from GRANDE import GRANDE\n",
    "\n",
    "params = {\n",
    "        'depth': 5, # tree depth\n",
    "        'n_estimators': 2048, # number of estimators / trees\n",
    "\n",
    "        'learning_rate_weights': 0.005, # learning rate for leaf weights\n",
    "        'learning_rate_index': 0.01, # learning rate for split indices\n",
    "        'learning_rate_values': 0.01, # learning rate for split values\n",
    "        'learning_rate_leaf': 0.01, # learning rate for leafs (logits)\n",
    "\n",
    "        'optimizer': 'adam', # optimizer\n",
    "        'cosine_decay_steps': 0, # decay steps for lr schedule (CosineDecayRestarts)\n",
    "\n",
    "        'loss': 'crossentropy', # loss function (default 'crossentropy' for binary & multi-class classification and 'mse' for regression)\n",
    "        'focal_loss': False, # use focal loss {True, False}\n",
    "        'temperature': 0.0, # temperature for stochastic re-weighted GD (0.0, 1.0)\n",
    "\n",
    "        'from_logits': True, # use logits for weighting {True, False}\n",
    "        'use_class_weights': True, # use class weights for training {True, False}\n",
    "\n",
    "        'dropout': 0.0, # dropout rate (here, dropout randomly disables individual estimators of the ensemble during training)\n",
    "\n",
    "        'selected_variables': 0.8, # feature subset percentage (0.0, 1.0)\n",
    "        'data_subset_fraction': 1.0, # data subset percentage (0.0, 1.0)\n",
    "}\n",
    "\n",
    "args = {\n",
    "    'epochs': 1_000, # number of epochs for training\n",
    "    'early_stopping_epochs': 25, # patience for early stopping (best weights are restored)\n",
    "    'batch_size': 64,  # batch size for training\n",
    "\n",
    "    'cat_idx': categorical_feature_indices, # put list of categorical indices\n",
    "    'objective': 'binary', # objective / task {'binary', 'classification', 'regression'}\n",
    "    \n",
    "    'random_seed': 42,\n",
    "    'verbose': 1,       \n",
    "}\n",
    "\n",
    "model_grande = GRANDE(params=params, args=args)\n",
    "\n",
    "model_grande.fit(X_train=X_train,\n",
    "          y_train=y_train,\n",
    "          X_val=X_valid,\n",
    "          y_val=y_valid)\n",
    "\n",
    "preds_grande = model_grande.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b251df9d-67f0-4dd1-a0cc-379621e33fad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:57:30.601353Z",
     "iopub.status.busy": "2024-03-15T11:57:30.601004Z",
     "iopub.status.idle": "2024-03-15T11:57:30.604147Z",
     "shell.execute_reply": "2024-03-15T11:57:30.603752Z",
     "shell.execute_reply.started": "2024-03-15T11:57:30.601338Z"
    }
   },
   "outputs": [],
   "source": [
    "def calculate_sample_weights(y_data):\n",
    "    class_weights = sklearn.utils.class_weight.compute_class_weight(class_weight = 'balanced', classes = np.unique(y_data), y = y_data)\n",
    "    sample_weights = sklearn.utils.class_weight.compute_sample_weight(class_weight = 'balanced', y =y_data)\n",
    "    return sample_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c17e805c-0110-4e39-97d3-34573804296b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:57:30.605592Z",
     "iopub.status.busy": "2024-03-15T11:57:30.605338Z",
     "iopub.status.idle": "2024-03-15T11:57:30.611465Z",
     "shell.execute_reply": "2024-03-15T11:57:30.611074Z",
     "shell.execute_reply.started": "2024-03-15T11:57:30.605578Z"
    }
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    y_train = y_train.values.codes.astype(np.float64)\n",
    "    y_valid = y_valid.values.codes.astype(np.float64)\n",
    "    y_test = y_test.values.codes.astype(np.float64)\n",
    "except:\n",
    "    y_train = y_train.values.astype(np.float64)\n",
    "    y_valid = y_valid.values.astype(np.float64)\n",
    "    y_test = y_test.values.astype(np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cfc121e3-2d7b-4fb1-8616-0318fcc20ac1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:57:30.612164Z",
     "iopub.status.busy": "2024-03-15T11:57:30.611960Z",
     "iopub.status.idle": "2024-03-15T11:57:30.627515Z",
     "shell.execute_reply": "2024-03-15T11:57:30.627108Z",
     "shell.execute_reply.started": "2024-03-15T11:57:30.612151Z"
    }
   },
   "outputs": [],
   "source": [
    "binary_indices = []\n",
    "low_cardinality_indices = []\n",
    "high_cardinality_indices = []\n",
    "num_columns = []\n",
    "for column_index, column in enumerate(X_train.columns):\n",
    "    if column_index in categorical_feature_indices:\n",
    "        if len(X_train.iloc[:,column_index].unique()) <= 2:\n",
    "            binary_indices.append(column)\n",
    "        if len(X_train.iloc[:,column_index].unique()) < 5:\n",
    "            low_cardinality_indices.append(column)\n",
    "        else:\n",
    "            high_cardinality_indices.append(column)\n",
    "    else:\n",
    "        num_columns.append(column)     \n",
    "cat_columns = [col for col in X_train.columns if col not in num_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f7e55785-bf56-4aad-9825-7c5737be5ab7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:57:30.628204Z",
     "iopub.status.busy": "2024-03-15T11:57:30.627992Z",
     "iopub.status.idle": "2024-03-15T11:57:30.691371Z",
     "shell.execute_reply": "2024-03-15T11:57:30.690745Z",
     "shell.execute_reply.started": "2024-03-15T11:57:30.628191Z"
    }
   },
   "outputs": [],
   "source": [
    "if len(num_columns) > 0:\n",
    "    mean_train_num = X_train[num_columns].mean(axis=0).iloc[0]\n",
    "    X_train[num_columns] = X_train[num_columns].fillna(mean_train_num)\n",
    "    X_valid[num_columns] = X_valid[num_columns].fillna(mean_train_num)\n",
    "    X_test[num_columns] = X_test[num_columns].fillna(mean_train_num)\n",
    "if len(cat_columns) > 0:\n",
    "    mode_train_cat = X_train[cat_columns].mode(axis=0).iloc[0]\n",
    "    X_train[cat_columns] = X_train[cat_columns].fillna(mode_train_cat)\n",
    "    X_valid[cat_columns] = X_valid[cat_columns].fillna(mode_train_cat)\n",
    "    X_test[cat_columns] = X_test[cat_columns].fillna(mode_train_cat)\n",
    "\n",
    "X_train_raw = X_train.copy()\n",
    "X_valid_raw = X_valid.copy()\n",
    "X_test_raw = X_test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b5d5f00f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:57:30.692101Z",
     "iopub.status.busy": "2024-03-15T11:57:30.691967Z",
     "iopub.status.idle": "2024-03-15T11:57:31.276758Z",
     "shell.execute_reply": "2024-03-15T11:57:31.276086Z",
     "shell.execute_reply.started": "2024-03-15T11:57:30.692089Z"
    }
   },
   "outputs": [],
   "source": [
    "encoder_ordinal = ce.OrdinalEncoder(cols=binary_indices)\n",
    "encoder_ordinal.fit(X_train)\n",
    "X_train = encoder_ordinal.transform(X_train)\n",
    "X_valid = encoder_ordinal.transform(X_valid)  \n",
    "X_test = encoder_ordinal.transform(X_test)       \n",
    "\n",
    "encoder = ce.LeaveOneOutEncoder(cols=high_cardinality_indices)\n",
    "encoder.fit(X_train, y_train)\n",
    "X_train = encoder.transform(X_train)\n",
    "X_valid = encoder.transform(X_valid)\n",
    "X_test = encoder.transform(X_test)\n",
    "\n",
    "encoder = ce.OneHotEncoder(cols=low_cardinality_indices)\n",
    "encoder.fit(X_train)\n",
    "X_train = encoder.transform(X_train)\n",
    "X_valid = encoder.transform(X_valid)\n",
    "X_test = encoder.transform(X_test)\n",
    "\n",
    "X_train = X_train.astype(np.float32)\n",
    "X_valid = X_valid.astype(np.float32)\n",
    "X_test = X_test.astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "361f7d9f-7e58-4bc1-8855-153b9e972742",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:57:31.277594Z",
     "iopub.status.busy": "2024-03-15T11:57:31.277439Z",
     "iopub.status.idle": "2024-03-15T11:57:31.575860Z",
     "shell.execute_reply": "2024-03-15T11:57:31.575224Z",
     "shell.execute_reply.started": "2024-03-15T11:57:31.277581Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-logloss:0.61817\n",
      "[1]\tvalidation_0-logloss:0.58949\n",
      "[2]\tvalidation_0-logloss:0.57913\n",
      "[3]\tvalidation_0-logloss:0.57102\n",
      "[4]\tvalidation_0-logloss:0.57202\n",
      "[5]\tvalidation_0-logloss:0.56968\n",
      "[6]\tvalidation_0-logloss:0.57447\n",
      "[7]\tvalidation_0-logloss:0.58700\n",
      "[8]\tvalidation_0-logloss:0.59624\n",
      "[9]\tvalidation_0-logloss:0.59631\n",
      "[10]\tvalidation_0-logloss:0.60544\n",
      "[11]\tvalidation_0-logloss:0.61715\n",
      "[12]\tvalidation_0-logloss:0.62893\n",
      "[13]\tvalidation_0-logloss:0.63263\n",
      "[14]\tvalidation_0-logloss:0.63697\n",
      "[15]\tvalidation_0-logloss:0.64000\n",
      "[16]\tvalidation_0-logloss:0.64394\n",
      "[17]\tvalidation_0-logloss:0.64549\n",
      "[18]\tvalidation_0-logloss:0.65777\n",
      "[19]\tvalidation_0-logloss:0.66242\n",
      "[20]\tvalidation_0-logloss:0.66363\n",
      "[21]\tvalidation_0-logloss:0.67369\n",
      "[22]\tvalidation_0-logloss:0.67608\n",
      "[23]\tvalidation_0-logloss:0.67905\n",
      "[24]\tvalidation_0-logloss:0.68762\n",
      "[25]\tvalidation_0-logloss:0.68740\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "model_xgb = XGBClassifier(n_estimators=1000, early_stopping_rounds=20)\n",
    "model_xgb.fit(X_train, \n",
    "                y_train, \n",
    "                sample_weight=calculate_sample_weights(y_train), \n",
    "                eval_set=[(X_valid, y_valid)], \n",
    "                sample_weight_eval_set=[calculate_sample_weights(y_valid)]\n",
    "             )\n",
    "if args['objective'] == 'regression':\n",
    "    preds_xgb = model_xgb.predict(X_test)\n",
    "else:\n",
    "    preds_xgb = model_xgb.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6b23c67e-cdde-40c2-842a-61383e1a62d0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:57:31.576940Z",
     "iopub.status.busy": "2024-03-15T11:57:31.576558Z",
     "iopub.status.idle": "2024-03-15T11:57:52.427379Z",
     "shell.execute_reply": "2024-03-15T11:57:52.426427Z",
     "shell.execute_reply.started": "2024-03-15T11:57:31.576923Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.047977\n",
      "0:\tlearn: 0.6738701\ttest: 0.6755301\tbest: 0.6755301 (0)\ttotal: 165ms\tremaining: 2m 44s\n",
      "1:\tlearn: 0.6590547\ttest: 0.6620922\tbest: 0.6620922 (1)\ttotal: 233ms\tremaining: 1m 56s\n",
      "2:\tlearn: 0.6438563\ttest: 0.6501098\tbest: 0.6501098 (2)\ttotal: 326ms\tremaining: 1m 48s\n",
      "3:\tlearn: 0.6281289\ttest: 0.6374755\tbest: 0.6374755 (3)\ttotal: 420ms\tremaining: 1m 44s\n",
      "4:\tlearn: 0.6134127\ttest: 0.6262980\tbest: 0.6262980 (4)\ttotal: 524ms\tremaining: 1m 44s\n",
      "5:\tlearn: 0.6051989\ttest: 0.6202962\tbest: 0.6202962 (5)\ttotal: 586ms\tremaining: 1m 37s\n",
      "6:\tlearn: 0.5932274\ttest: 0.6109070\tbest: 0.6109070 (6)\ttotal: 687ms\tremaining: 1m 37s\n",
      "7:\tlearn: 0.5848141\ttest: 0.6042882\tbest: 0.6042882 (7)\ttotal: 782ms\tremaining: 1m 36s\n",
      "8:\tlearn: 0.5757846\ttest: 0.5981882\tbest: 0.5981882 (8)\ttotal: 880ms\tremaining: 1m 36s\n",
      "9:\tlearn: 0.5662888\ttest: 0.5905358\tbest: 0.5905358 (9)\ttotal: 982ms\tremaining: 1m 37s\n",
      "10:\tlearn: 0.5589223\ttest: 0.5846889\tbest: 0.5846889 (10)\ttotal: 1.08s\tremaining: 1m 36s\n",
      "11:\tlearn: 0.5508372\ttest: 0.5783983\tbest: 0.5783983 (11)\ttotal: 1.18s\tremaining: 1m 37s\n",
      "12:\tlearn: 0.5447033\ttest: 0.5743001\tbest: 0.5743001 (12)\ttotal: 1.28s\tremaining: 1m 36s\n",
      "13:\tlearn: 0.5402192\ttest: 0.5700308\tbest: 0.5700308 (13)\ttotal: 1.37s\tremaining: 1m 36s\n",
      "14:\tlearn: 0.5351919\ttest: 0.5668324\tbest: 0.5668324 (14)\ttotal: 1.47s\tremaining: 1m 36s\n",
      "15:\tlearn: 0.5301341\ttest: 0.5637916\tbest: 0.5637916 (15)\ttotal: 1.57s\tremaining: 1m 36s\n",
      "16:\tlearn: 0.5273010\ttest: 0.5614049\tbest: 0.5614049 (16)\ttotal: 1.61s\tremaining: 1m 33s\n",
      "17:\tlearn: 0.5231602\ttest: 0.5588854\tbest: 0.5588854 (17)\ttotal: 1.72s\tremaining: 1m 33s\n",
      "18:\tlearn: 0.5174098\ttest: 0.5560311\tbest: 0.5560311 (18)\ttotal: 1.81s\tremaining: 1m 33s\n",
      "19:\tlearn: 0.5136603\ttest: 0.5527964\tbest: 0.5527964 (19)\ttotal: 1.91s\tremaining: 1m 33s\n",
      "20:\tlearn: 0.5097790\ttest: 0.5501795\tbest: 0.5501795 (20)\ttotal: 2.01s\tremaining: 1m 33s\n",
      "21:\tlearn: 0.5068590\ttest: 0.5486268\tbest: 0.5486268 (21)\ttotal: 2.11s\tremaining: 1m 33s\n",
      "22:\tlearn: 0.5027830\ttest: 0.5458326\tbest: 0.5458326 (22)\ttotal: 2.22s\tremaining: 1m 34s\n",
      "23:\tlearn: 0.4988387\ttest: 0.5443584\tbest: 0.5443584 (23)\ttotal: 2.32s\tremaining: 1m 34s\n",
      "24:\tlearn: 0.4965991\ttest: 0.5427437\tbest: 0.5427437 (24)\ttotal: 2.4s\tremaining: 1m 33s\n",
      "25:\tlearn: 0.4935131\ttest: 0.5405988\tbest: 0.5405988 (25)\ttotal: 2.51s\tremaining: 1m 34s\n",
      "26:\tlearn: 0.4920190\ttest: 0.5400687\tbest: 0.5400687 (26)\ttotal: 2.55s\tremaining: 1m 32s\n",
      "27:\tlearn: 0.4895990\ttest: 0.5386297\tbest: 0.5386297 (27)\ttotal: 2.65s\tremaining: 1m 32s\n",
      "28:\tlearn: 0.4867917\ttest: 0.5376932\tbest: 0.5376932 (28)\ttotal: 2.75s\tremaining: 1m 32s\n",
      "29:\tlearn: 0.4840789\ttest: 0.5369178\tbest: 0.5369178 (29)\ttotal: 2.85s\tremaining: 1m 32s\n",
      "30:\tlearn: 0.4816525\ttest: 0.5360564\tbest: 0.5360564 (30)\ttotal: 2.96s\tremaining: 1m 32s\n",
      "31:\tlearn: 0.4799453\ttest: 0.5355162\tbest: 0.5355162 (31)\ttotal: 3.05s\tremaining: 1m 32s\n",
      "32:\tlearn: 0.4771746\ttest: 0.5340567\tbest: 0.5340567 (32)\ttotal: 3.15s\tremaining: 1m 32s\n",
      "33:\tlearn: 0.4755385\ttest: 0.5337042\tbest: 0.5337042 (33)\ttotal: 3.25s\tremaining: 1m 32s\n",
      "34:\tlearn: 0.4743555\ttest: 0.5324148\tbest: 0.5324148 (34)\ttotal: 3.35s\tremaining: 1m 32s\n",
      "35:\tlearn: 0.4724657\ttest: 0.5307660\tbest: 0.5307660 (35)\ttotal: 3.45s\tremaining: 1m 32s\n",
      "36:\tlearn: 0.4698649\ttest: 0.5291584\tbest: 0.5291584 (36)\ttotal: 3.56s\tremaining: 1m 32s\n",
      "37:\tlearn: 0.4682657\ttest: 0.5283045\tbest: 0.5283045 (37)\ttotal: 3.66s\tremaining: 1m 32s\n",
      "38:\tlearn: 0.4668524\ttest: 0.5279314\tbest: 0.5279314 (38)\ttotal: 3.76s\tremaining: 1m 32s\n",
      "39:\tlearn: 0.4646543\ttest: 0.5270420\tbest: 0.5270420 (39)\ttotal: 3.86s\tremaining: 1m 32s\n",
      "40:\tlearn: 0.4632387\ttest: 0.5260156\tbest: 0.5260156 (40)\ttotal: 3.96s\tremaining: 1m 32s\n",
      "41:\tlearn: 0.4620355\ttest: 0.5252222\tbest: 0.5252222 (41)\ttotal: 4.07s\tremaining: 1m 32s\n",
      "42:\tlearn: 0.4606320\ttest: 0.5242838\tbest: 0.5242838 (42)\ttotal: 4.17s\tremaining: 1m 32s\n",
      "43:\tlearn: 0.4591662\ttest: 0.5236499\tbest: 0.5236499 (43)\ttotal: 4.26s\tremaining: 1m 32s\n",
      "44:\tlearn: 0.4577533\ttest: 0.5232656\tbest: 0.5232656 (44)\ttotal: 4.37s\tremaining: 1m 32s\n",
      "45:\tlearn: 0.4558061\ttest: 0.5224758\tbest: 0.5224758 (45)\ttotal: 4.46s\tremaining: 1m 32s\n",
      "46:\tlearn: 0.4539798\ttest: 0.5217470\tbest: 0.5217470 (46)\ttotal: 4.57s\tremaining: 1m 32s\n",
      "47:\tlearn: 0.4522604\ttest: 0.5221785\tbest: 0.5217470 (46)\ttotal: 4.67s\tremaining: 1m 32s\n",
      "48:\tlearn: 0.4508624\ttest: 0.5216429\tbest: 0.5216429 (48)\ttotal: 4.77s\tremaining: 1m 32s\n",
      "49:\tlearn: 0.4493435\ttest: 0.5208824\tbest: 0.5208824 (49)\ttotal: 4.87s\tremaining: 1m 32s\n",
      "50:\tlearn: 0.4484078\ttest: 0.5203980\tbest: 0.5203980 (50)\ttotal: 4.97s\tremaining: 1m 32s\n",
      "51:\tlearn: 0.4460948\ttest: 0.5193948\tbest: 0.5193948 (51)\ttotal: 5.07s\tremaining: 1m 32s\n",
      "52:\tlearn: 0.4447458\ttest: 0.5195007\tbest: 0.5193948 (51)\ttotal: 5.17s\tremaining: 1m 32s\n",
      "53:\tlearn: 0.4428160\ttest: 0.5193171\tbest: 0.5193171 (53)\ttotal: 5.27s\tremaining: 1m 32s\n",
      "54:\tlearn: 0.4412406\ttest: 0.5192006\tbest: 0.5192006 (54)\ttotal: 5.37s\tremaining: 1m 32s\n",
      "55:\tlearn: 0.4400487\ttest: 0.5188502\tbest: 0.5188502 (55)\ttotal: 5.47s\tremaining: 1m 32s\n",
      "56:\tlearn: 0.4386987\ttest: 0.5182503\tbest: 0.5182503 (56)\ttotal: 5.59s\tremaining: 1m 32s\n",
      "57:\tlearn: 0.4377159\ttest: 0.5182941\tbest: 0.5182503 (56)\ttotal: 5.7s\tremaining: 1m 32s\n",
      "58:\tlearn: 0.4372125\ttest: 0.5181658\tbest: 0.5181658 (58)\ttotal: 5.8s\tremaining: 1m 32s\n",
      "59:\tlearn: 0.4358697\ttest: 0.5178906\tbest: 0.5178906 (59)\ttotal: 5.91s\tremaining: 1m 32s\n",
      "60:\tlearn: 0.4348061\ttest: 0.5172451\tbest: 0.5172451 (60)\ttotal: 6.01s\tremaining: 1m 32s\n",
      "61:\tlearn: 0.4336819\ttest: 0.5170574\tbest: 0.5170574 (61)\ttotal: 6.11s\tremaining: 1m 32s\n",
      "62:\tlearn: 0.4331637\ttest: 0.5166699\tbest: 0.5166699 (62)\ttotal: 6.21s\tremaining: 1m 32s\n",
      "63:\tlearn: 0.4318591\ttest: 0.5159768\tbest: 0.5159768 (63)\ttotal: 6.31s\tremaining: 1m 32s\n",
      "64:\tlearn: 0.4303671\ttest: 0.5156643\tbest: 0.5156643 (64)\ttotal: 6.42s\tremaining: 1m 32s\n",
      "65:\tlearn: 0.4294351\ttest: 0.5154294\tbest: 0.5154294 (65)\ttotal: 6.53s\tremaining: 1m 32s\n",
      "66:\tlearn: 0.4289233\ttest: 0.5151205\tbest: 0.5151205 (66)\ttotal: 6.61s\tremaining: 1m 32s\n",
      "67:\tlearn: 0.4278221\ttest: 0.5153206\tbest: 0.5151205 (66)\ttotal: 6.71s\tremaining: 1m 32s\n",
      "68:\tlearn: 0.4266775\ttest: 0.5150592\tbest: 0.5150592 (68)\ttotal: 6.82s\tremaining: 1m 31s\n",
      "69:\tlearn: 0.4253312\ttest: 0.5147168\tbest: 0.5147168 (69)\ttotal: 6.91s\tremaining: 1m 31s\n",
      "70:\tlearn: 0.4239908\ttest: 0.5142156\tbest: 0.5142156 (70)\ttotal: 7.01s\tremaining: 1m 31s\n",
      "71:\tlearn: 0.4239078\ttest: 0.5141317\tbest: 0.5141317 (71)\ttotal: 7.07s\tremaining: 1m 31s\n",
      "72:\tlearn: 0.4237689\ttest: 0.5139263\tbest: 0.5139263 (72)\ttotal: 7.1s\tremaining: 1m 30s\n",
      "73:\tlearn: 0.4228262\ttest: 0.5139126\tbest: 0.5139126 (73)\ttotal: 7.18s\tremaining: 1m 29s\n",
      "74:\tlearn: 0.4221208\ttest: 0.5137374\tbest: 0.5137374 (74)\ttotal: 7.28s\tremaining: 1m 29s\n",
      "75:\tlearn: 0.4217107\ttest: 0.5134631\tbest: 0.5134631 (75)\ttotal: 7.34s\tremaining: 1m 29s\n",
      "76:\tlearn: 0.4206589\ttest: 0.5140551\tbest: 0.5134631 (75)\ttotal: 7.45s\tremaining: 1m 29s\n",
      "77:\tlearn: 0.4196407\ttest: 0.5143257\tbest: 0.5134631 (75)\ttotal: 7.55s\tremaining: 1m 29s\n",
      "78:\tlearn: 0.4188319\ttest: 0.5142454\tbest: 0.5134631 (75)\ttotal: 7.65s\tremaining: 1m 29s\n",
      "79:\tlearn: 0.4176493\ttest: 0.5142260\tbest: 0.5134631 (75)\ttotal: 7.75s\tremaining: 1m 29s\n",
      "80:\tlearn: 0.4170162\ttest: 0.5142153\tbest: 0.5134631 (75)\ttotal: 7.84s\tremaining: 1m 28s\n",
      "81:\tlearn: 0.4162356\ttest: 0.5144343\tbest: 0.5134631 (75)\ttotal: 7.94s\tremaining: 1m 28s\n",
      "82:\tlearn: 0.4150635\ttest: 0.5147101\tbest: 0.5134631 (75)\ttotal: 8.05s\tremaining: 1m 28s\n",
      "83:\tlearn: 0.4141552\ttest: 0.5146635\tbest: 0.5134631 (75)\ttotal: 8.15s\tremaining: 1m 28s\n",
      "84:\tlearn: 0.4132422\ttest: 0.5143741\tbest: 0.5134631 (75)\ttotal: 8.26s\tremaining: 1m 28s\n",
      "85:\tlearn: 0.4130360\ttest: 0.5142130\tbest: 0.5134631 (75)\ttotal: 8.31s\tremaining: 1m 28s\n",
      "86:\tlearn: 0.4127646\ttest: 0.5141590\tbest: 0.5134631 (75)\ttotal: 8.39s\tremaining: 1m 28s\n",
      "87:\tlearn: 0.4120442\ttest: 0.5136968\tbest: 0.5134631 (75)\ttotal: 8.49s\tremaining: 1m 28s\n",
      "88:\tlearn: 0.4117758\ttest: 0.5135610\tbest: 0.5134631 (75)\ttotal: 8.59s\tremaining: 1m 27s\n",
      "89:\tlearn: 0.4108341\ttest: 0.5134290\tbest: 0.5134290 (89)\ttotal: 8.67s\tremaining: 1m 27s\n",
      "90:\tlearn: 0.4091175\ttest: 0.5132143\tbest: 0.5132143 (90)\ttotal: 8.78s\tremaining: 1m 27s\n",
      "91:\tlearn: 0.4083320\ttest: 0.5132633\tbest: 0.5132143 (90)\ttotal: 8.88s\tremaining: 1m 27s\n",
      "92:\tlearn: 0.4076566\ttest: 0.5128089\tbest: 0.5128089 (92)\ttotal: 8.98s\tremaining: 1m 27s\n",
      "93:\tlearn: 0.4069641\ttest: 0.5125266\tbest: 0.5125266 (93)\ttotal: 9.09s\tremaining: 1m 27s\n",
      "94:\tlearn: 0.4056949\ttest: 0.5119460\tbest: 0.5119460 (94)\ttotal: 9.19s\tremaining: 1m 27s\n",
      "95:\tlearn: 0.4046216\ttest: 0.5122375\tbest: 0.5119460 (94)\ttotal: 9.3s\tremaining: 1m 27s\n",
      "96:\tlearn: 0.4037038\ttest: 0.5118532\tbest: 0.5118532 (96)\ttotal: 9.39s\tremaining: 1m 27s\n",
      "97:\tlearn: 0.4035115\ttest: 0.5117283\tbest: 0.5117283 (97)\ttotal: 9.49s\tremaining: 1m 27s\n",
      "98:\tlearn: 0.4030275\ttest: 0.5115880\tbest: 0.5115880 (98)\ttotal: 9.6s\tremaining: 1m 27s\n",
      "99:\tlearn: 0.4021809\ttest: 0.5112226\tbest: 0.5112226 (99)\ttotal: 9.69s\tremaining: 1m 27s\n",
      "100:\tlearn: 0.4015090\ttest: 0.5115030\tbest: 0.5112226 (99)\ttotal: 9.8s\tremaining: 1m 27s\n",
      "101:\tlearn: 0.4010460\ttest: 0.5115215\tbest: 0.5112226 (99)\ttotal: 9.89s\tremaining: 1m 27s\n",
      "102:\tlearn: 0.4008657\ttest: 0.5115077\tbest: 0.5112226 (99)\ttotal: 10s\tremaining: 1m 27s\n",
      "103:\tlearn: 0.4001509\ttest: 0.5108451\tbest: 0.5108451 (103)\ttotal: 10.1s\tremaining: 1m 27s\n",
      "104:\tlearn: 0.3995489\ttest: 0.5106618\tbest: 0.5106618 (104)\ttotal: 10.2s\tremaining: 1m 27s\n",
      "105:\tlearn: 0.3991027\ttest: 0.5103941\tbest: 0.5103941 (105)\ttotal: 10.3s\tremaining: 1m 27s\n",
      "106:\tlearn: 0.3983257\ttest: 0.5107081\tbest: 0.5103941 (105)\ttotal: 10.4s\tremaining: 1m 27s\n",
      "107:\tlearn: 0.3973440\ttest: 0.5104172\tbest: 0.5103941 (105)\ttotal: 10.5s\tremaining: 1m 26s\n",
      "108:\tlearn: 0.3972470\ttest: 0.5103407\tbest: 0.5103407 (108)\ttotal: 10.6s\tremaining: 1m 26s\n",
      "109:\tlearn: 0.3961923\ttest: 0.5098895\tbest: 0.5098895 (109)\ttotal: 10.7s\tremaining: 1m 26s\n",
      "110:\tlearn: 0.3956434\ttest: 0.5096094\tbest: 0.5096094 (110)\ttotal: 10.8s\tremaining: 1m 26s\n",
      "111:\tlearn: 0.3943166\ttest: 0.5100694\tbest: 0.5096094 (110)\ttotal: 10.9s\tremaining: 1m 26s\n",
      "112:\tlearn: 0.3939594\ttest: 0.5102020\tbest: 0.5096094 (110)\ttotal: 10.9s\tremaining: 1m 25s\n",
      "113:\tlearn: 0.3928364\ttest: 0.5096822\tbest: 0.5096094 (110)\ttotal: 11s\tremaining: 1m 25s\n",
      "114:\tlearn: 0.3915690\ttest: 0.5102480\tbest: 0.5096094 (110)\ttotal: 11.2s\tremaining: 1m 25s\n",
      "115:\tlearn: 0.3904765\ttest: 0.5098315\tbest: 0.5096094 (110)\ttotal: 11.3s\tremaining: 1m 25s\n",
      "116:\tlearn: 0.3899511\ttest: 0.5100134\tbest: 0.5096094 (110)\ttotal: 11.3s\tremaining: 1m 25s\n",
      "117:\tlearn: 0.3892750\ttest: 0.5100222\tbest: 0.5096094 (110)\ttotal: 11.5s\tremaining: 1m 25s\n",
      "118:\tlearn: 0.3885256\ttest: 0.5100760\tbest: 0.5096094 (110)\ttotal: 11.6s\tremaining: 1m 25s\n",
      "119:\tlearn: 0.3879885\ttest: 0.5101717\tbest: 0.5096094 (110)\ttotal: 11.7s\tremaining: 1m 25s\n",
      "120:\tlearn: 0.3877867\ttest: 0.5102002\tbest: 0.5096094 (110)\ttotal: 11.8s\tremaining: 1m 25s\n",
      "121:\tlearn: 0.3871158\ttest: 0.5099259\tbest: 0.5096094 (110)\ttotal: 11.9s\tremaining: 1m 25s\n",
      "122:\tlearn: 0.3866180\ttest: 0.5101057\tbest: 0.5096094 (110)\ttotal: 12s\tremaining: 1m 25s\n",
      "123:\tlearn: 0.3863558\ttest: 0.5101695\tbest: 0.5096094 (110)\ttotal: 12.1s\tremaining: 1m 25s\n",
      "124:\tlearn: 0.3854500\ttest: 0.5099909\tbest: 0.5096094 (110)\ttotal: 12.2s\tremaining: 1m 25s\n",
      "125:\tlearn: 0.3849377\ttest: 0.5101998\tbest: 0.5096094 (110)\ttotal: 12.3s\tremaining: 1m 25s\n",
      "126:\tlearn: 0.3840049\ttest: 0.5100882\tbest: 0.5096094 (110)\ttotal: 12.4s\tremaining: 1m 25s\n",
      "127:\tlearn: 0.3830857\ttest: 0.5100424\tbest: 0.5096094 (110)\ttotal: 12.5s\tremaining: 1m 25s\n",
      "128:\tlearn: 0.3824818\ttest: 0.5098228\tbest: 0.5096094 (110)\ttotal: 12.6s\tremaining: 1m 25s\n",
      "129:\tlearn: 0.3818878\ttest: 0.5097249\tbest: 0.5096094 (110)\ttotal: 12.7s\tremaining: 1m 25s\n",
      "130:\tlearn: 0.3812166\ttest: 0.5089894\tbest: 0.5089894 (130)\ttotal: 12.8s\tremaining: 1m 24s\n",
      "131:\tlearn: 0.3804604\ttest: 0.5087992\tbest: 0.5087992 (131)\ttotal: 12.9s\tremaining: 1m 24s\n",
      "132:\tlearn: 0.3798619\ttest: 0.5089803\tbest: 0.5087992 (131)\ttotal: 13s\tremaining: 1m 24s\n",
      "133:\tlearn: 0.3793899\ttest: 0.5090583\tbest: 0.5087992 (131)\ttotal: 13.1s\tremaining: 1m 24s\n",
      "134:\tlearn: 0.3789654\ttest: 0.5088810\tbest: 0.5087992 (131)\ttotal: 13.2s\tremaining: 1m 24s\n",
      "135:\tlearn: 0.3781716\ttest: 0.5083386\tbest: 0.5083386 (135)\ttotal: 13.4s\tremaining: 1m 24s\n",
      "136:\tlearn: 0.3774964\ttest: 0.5082287\tbest: 0.5082287 (136)\ttotal: 13.5s\tremaining: 1m 24s\n",
      "137:\tlearn: 0.3769306\ttest: 0.5082420\tbest: 0.5082287 (136)\ttotal: 13.6s\tremaining: 1m 24s\n",
      "138:\tlearn: 0.3763334\ttest: 0.5082716\tbest: 0.5082287 (136)\ttotal: 13.7s\tremaining: 1m 24s\n",
      "139:\tlearn: 0.3756023\ttest: 0.5090553\tbest: 0.5082287 (136)\ttotal: 13.8s\tremaining: 1m 24s\n",
      "140:\tlearn: 0.3744634\ttest: 0.5087139\tbest: 0.5082287 (136)\ttotal: 13.9s\tremaining: 1m 24s\n",
      "141:\tlearn: 0.3738744\ttest: 0.5084388\tbest: 0.5082287 (136)\ttotal: 14s\tremaining: 1m 24s\n",
      "142:\tlearn: 0.3733198\ttest: 0.5086448\tbest: 0.5082287 (136)\ttotal: 14.1s\tremaining: 1m 24s\n",
      "143:\tlearn: 0.3725403\ttest: 0.5092001\tbest: 0.5082287 (136)\ttotal: 14.2s\tremaining: 1m 24s\n",
      "144:\tlearn: 0.3720424\ttest: 0.5091780\tbest: 0.5082287 (136)\ttotal: 14.3s\tremaining: 1m 24s\n",
      "145:\tlearn: 0.3710328\ttest: 0.5090606\tbest: 0.5082287 (136)\ttotal: 14.4s\tremaining: 1m 24s\n",
      "146:\tlearn: 0.3709490\ttest: 0.5090941\tbest: 0.5082287 (136)\ttotal: 14.5s\tremaining: 1m 24s\n",
      "147:\tlearn: 0.3698424\ttest: 0.5087504\tbest: 0.5082287 (136)\ttotal: 14.6s\tremaining: 1m 24s\n",
      "148:\tlearn: 0.3693312\ttest: 0.5085800\tbest: 0.5082287 (136)\ttotal: 14.7s\tremaining: 1m 24s\n",
      "149:\tlearn: 0.3688429\ttest: 0.5086697\tbest: 0.5082287 (136)\ttotal: 14.8s\tremaining: 1m 24s\n",
      "150:\tlearn: 0.3685298\ttest: 0.5086198\tbest: 0.5082287 (136)\ttotal: 15s\tremaining: 1m 24s\n",
      "151:\tlearn: 0.3676950\ttest: 0.5083866\tbest: 0.5082287 (136)\ttotal: 15.1s\tremaining: 1m 24s\n",
      "152:\tlearn: 0.3665181\ttest: 0.5079430\tbest: 0.5079430 (152)\ttotal: 15.2s\tremaining: 1m 23s\n",
      "153:\tlearn: 0.3657733\ttest: 0.5083545\tbest: 0.5079430 (152)\ttotal: 15.3s\tremaining: 1m 23s\n",
      "154:\tlearn: 0.3652710\ttest: 0.5079265\tbest: 0.5079265 (154)\ttotal: 15.4s\tremaining: 1m 23s\n",
      "155:\tlearn: 0.3648924\ttest: 0.5079489\tbest: 0.5079265 (154)\ttotal: 15.5s\tremaining: 1m 23s\n",
      "156:\tlearn: 0.3637806\ttest: 0.5074084\tbest: 0.5074084 (156)\ttotal: 15.6s\tremaining: 1m 23s\n",
      "157:\tlearn: 0.3632405\ttest: 0.5073191\tbest: 0.5073191 (157)\ttotal: 15.7s\tremaining: 1m 23s\n",
      "158:\tlearn: 0.3623911\ttest: 0.5068061\tbest: 0.5068061 (158)\ttotal: 15.8s\tremaining: 1m 23s\n",
      "159:\tlearn: 0.3621138\ttest: 0.5067240\tbest: 0.5067240 (159)\ttotal: 15.9s\tremaining: 1m 23s\n",
      "160:\tlearn: 0.3611554\ttest: 0.5069481\tbest: 0.5067240 (159)\ttotal: 16s\tremaining: 1m 23s\n",
      "161:\tlearn: 0.3598932\ttest: 0.5072188\tbest: 0.5067240 (159)\ttotal: 16.1s\tremaining: 1m 23s\n",
      "162:\tlearn: 0.3597427\ttest: 0.5070021\tbest: 0.5067240 (159)\ttotal: 16.2s\tremaining: 1m 23s\n",
      "163:\tlearn: 0.3590274\ttest: 0.5073152\tbest: 0.5067240 (159)\ttotal: 16.3s\tremaining: 1m 22s\n",
      "164:\tlearn: 0.3576039\ttest: 0.5066460\tbest: 0.5066460 (164)\ttotal: 16.4s\tremaining: 1m 22s\n",
      "165:\tlearn: 0.3569895\ttest: 0.5067082\tbest: 0.5066460 (164)\ttotal: 16.5s\tremaining: 1m 22s\n",
      "166:\tlearn: 0.3565255\ttest: 0.5065910\tbest: 0.5065910 (166)\ttotal: 16.6s\tremaining: 1m 22s\n",
      "167:\tlearn: 0.3556883\ttest: 0.5064032\tbest: 0.5064032 (167)\ttotal: 16.7s\tremaining: 1m 22s\n",
      "168:\tlearn: 0.3545941\ttest: 0.5064414\tbest: 0.5064032 (167)\ttotal: 16.8s\tremaining: 1m 22s\n",
      "169:\tlearn: 0.3540935\ttest: 0.5060839\tbest: 0.5060839 (169)\ttotal: 16.9s\tremaining: 1m 22s\n",
      "170:\tlearn: 0.3535427\ttest: 0.5060219\tbest: 0.5060219 (170)\ttotal: 17s\tremaining: 1m 22s\n",
      "171:\tlearn: 0.3530170\ttest: 0.5061560\tbest: 0.5060219 (170)\ttotal: 17.1s\tremaining: 1m 22s\n",
      "172:\tlearn: 0.3525071\ttest: 0.5063416\tbest: 0.5060219 (170)\ttotal: 17.2s\tremaining: 1m 22s\n",
      "173:\tlearn: 0.3520189\ttest: 0.5062745\tbest: 0.5060219 (170)\ttotal: 17.3s\tremaining: 1m 22s\n",
      "174:\tlearn: 0.3510690\ttest: 0.5060421\tbest: 0.5060219 (170)\ttotal: 17.4s\tremaining: 1m 22s\n",
      "175:\tlearn: 0.3503487\ttest: 0.5060002\tbest: 0.5060002 (175)\ttotal: 17.6s\tremaining: 1m 22s\n",
      "176:\tlearn: 0.3495787\ttest: 0.5061916\tbest: 0.5060002 (175)\ttotal: 17.7s\tremaining: 1m 22s\n",
      "177:\tlearn: 0.3484528\ttest: 0.5053065\tbest: 0.5053065 (177)\ttotal: 17.8s\tremaining: 1m 22s\n",
      "178:\tlearn: 0.3478702\ttest: 0.5053716\tbest: 0.5053065 (177)\ttotal: 17.9s\tremaining: 1m 21s\n",
      "179:\tlearn: 0.3470374\ttest: 0.5057235\tbest: 0.5053065 (177)\ttotal: 18s\tremaining: 1m 21s\n",
      "180:\tlearn: 0.3463041\ttest: 0.5057950\tbest: 0.5053065 (177)\ttotal: 18.1s\tremaining: 1m 21s\n",
      "181:\tlearn: 0.3450065\ttest: 0.5060521\tbest: 0.5053065 (177)\ttotal: 18.2s\tremaining: 1m 21s\n",
      "182:\tlearn: 0.3444901\ttest: 0.5058767\tbest: 0.5053065 (177)\ttotal: 18.3s\tremaining: 1m 21s\n",
      "183:\tlearn: 0.3436147\ttest: 0.5063084\tbest: 0.5053065 (177)\ttotal: 18.4s\tremaining: 1m 21s\n",
      "184:\tlearn: 0.3434317\ttest: 0.5064843\tbest: 0.5053065 (177)\ttotal: 18.5s\tremaining: 1m 21s\n",
      "185:\tlearn: 0.3425579\ttest: 0.5059915\tbest: 0.5053065 (177)\ttotal: 18.6s\tremaining: 1m 21s\n",
      "186:\tlearn: 0.3422994\ttest: 0.5058662\tbest: 0.5053065 (177)\ttotal: 18.7s\tremaining: 1m 21s\n",
      "187:\tlearn: 0.3418043\ttest: 0.5058788\tbest: 0.5053065 (177)\ttotal: 18.8s\tremaining: 1m 21s\n",
      "188:\tlearn: 0.3407382\ttest: 0.5065781\tbest: 0.5053065 (177)\ttotal: 18.9s\tremaining: 1m 21s\n",
      "189:\tlearn: 0.3394479\ttest: 0.5070881\tbest: 0.5053065 (177)\ttotal: 19s\tremaining: 1m 21s\n",
      "190:\tlearn: 0.3389152\ttest: 0.5070578\tbest: 0.5053065 (177)\ttotal: 19.1s\tremaining: 1m 21s\n",
      "191:\tlearn: 0.3386440\ttest: 0.5068382\tbest: 0.5053065 (177)\ttotal: 19.3s\tremaining: 1m 21s\n",
      "192:\tlearn: 0.3386128\ttest: 0.5068467\tbest: 0.5053065 (177)\ttotal: 19.4s\tremaining: 1m 20s\n",
      "193:\tlearn: 0.3379356\ttest: 0.5070434\tbest: 0.5053065 (177)\ttotal: 19.5s\tremaining: 1m 20s\n",
      "194:\tlearn: 0.3372163\ttest: 0.5073649\tbest: 0.5053065 (177)\ttotal: 19.6s\tremaining: 1m 20s\n",
      "195:\tlearn: 0.3369111\ttest: 0.5073930\tbest: 0.5053065 (177)\ttotal: 19.7s\tremaining: 1m 20s\n",
      "196:\tlearn: 0.3362545\ttest: 0.5074880\tbest: 0.5053065 (177)\ttotal: 19.8s\tremaining: 1m 20s\n",
      "197:\tlearn: 0.3349509\ttest: 0.5078499\tbest: 0.5053065 (177)\ttotal: 19.9s\tremaining: 1m 20s\n",
      "Stopped by overfitting detector  (20 iterations wait)\n",
      "\n",
      "bestTest = 0.5053065341\n",
      "bestIteration = 177\n",
      "\n",
      "Shrink model to first 178 iterations.\n"
     ]
    }
   ],
   "source": [
    "from catboost import CatBoostClassifier, Pool\n",
    "\n",
    "model_catboost = CatBoostClassifier(n_estimators=1000, \n",
    "                                    early_stopping_rounds=20)\n",
    "train_data = Pool(\n",
    "        data=X_train_raw,\n",
    "        label=y_train,\n",
    "        cat_features=categorical_feature_indices,\n",
    "        weight=calculate_sample_weights(y_train)\n",
    "    )\n",
    "\n",
    "eval_data = Pool(\n",
    "        data=X_valid_raw,\n",
    "        label=y_valid,\n",
    "        cat_features=categorical_feature_indices,\n",
    "        weight=calculate_sample_weights(y_valid),\n",
    "    )\n",
    "\n",
    "model_catboost.fit(X=train_data, \n",
    "                   eval_set=eval_data)\n",
    "\n",
    "if args['objective'] == 'regression':\n",
    "    preds_catboost = model_catboost.predict(X_test_raw)\n",
    "else:\n",
    "    preds_catboost = model_catboost.predict_proba(X_test_raw)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "21b9be01-ba30-4300-b3e0-f4fd43b55c87",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:57:52.429019Z",
     "iopub.status.busy": "2024-03-15T11:57:52.428555Z",
     "iopub.status.idle": "2024-03-15T11:57:52.454151Z",
     "shell.execute_reply": "2024-03-15T11:57:52.453620Z",
     "shell.execute_reply.started": "2024-03-15T11:57:52.428990Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy GRANDE: 0.8436754176610979\n",
      "F1 Score GRANDE: 0.6910473006250651\n",
      "ROC AUC GRANDE: 0.8316313092165171\n",
      "\n",
      "\n",
      "Accuracy XGB: 0.7708830548926014\n",
      "F1 Score XGB: 0.6814621823451619\n",
      "ROC AUC XGB: 0.8041179702139509\n",
      "\n",
      "\n",
      "Accuracy CatBoost: 0.7905727923627685\n",
      "F1 Score CatBoost: 0.7106101637384612\n",
      "ROC AUC CatBoost: 0.8575216421710652\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if args['objective'] == 'binary':\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.round(preds_grande[:,1]))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.round(preds_grande[:,1]), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_grande[:,1], average='macro', multi_class='ovo')\n",
    "\n",
    "    print('Accuracy GRANDE:', accuracy)\n",
    "    print('F1 Score GRANDE:', f1_score)\n",
    "    print('ROC AUC GRANDE:', roc_auc)\n",
    "    print('\\n')\n",
    "\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.round(preds_xgb[:,1]))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.round(preds_xgb[:,1]), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_xgb[:,1], average='macro', multi_class='ovo')\n",
    "\n",
    "    print('Accuracy XGB:', accuracy)\n",
    "    print('F1 Score XGB:', f1_score)\n",
    "    print('ROC AUC XGB:', roc_auc)\n",
    "    print('\\n')\n",
    "\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.round(preds_catboost[:,1]))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.round(preds_catboost[:,1]), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_catboost[:,1], average='macro', multi_class='ovo')\n",
    "\n",
    "    print('Accuracy CatBoost:', accuracy)\n",
    "    print('F1 Score CatBoost:', f1_score)\n",
    "    print('ROC AUC CatBoost:', roc_auc)\n",
    "    print('\\n')\n",
    "elif args['objective'] == 'classification':\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.argmax(preds_grande, axis=1))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.argmax(preds_grande, axis=1), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_grande, average='macro', multi_class='ovo', labels=[i for i in range(preds_grande.shape[1])])\n",
    "\n",
    "    print('Accuracy GRANDE:', accuracy)\n",
    "    print('F1 Score GRANDE:', f1_score)\n",
    "    print('ROC AUC GRANDE:', roc_auc)\n",
    "    print('\\n')\n",
    "\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.argmax(preds_xgb, axis=1))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.argmax(preds_xgb, axis=1), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_xgb, average='macro', multi_class='ovo', labels=[i for i in range(preds_grande.shape[1])])\n",
    "\n",
    "    print('Accuracy XGB:', accuracy)\n",
    "    print('F1 Score XGB:', f1_score)\n",
    "    print('ROC AUC XGB:', roc_auc)\n",
    "    print('\\n')\n",
    "\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.argmax(preds_catboost, axis=1))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.argmax(preds_catboost, axis=1), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_catboost, average='macro', multi_class='ovo', labels=[i for i in range(preds_grande.shape[1])])\n",
    "\n",
    "    print('Accuracy CatBoost:', accuracy)\n",
    "    print('F1 Score CatBoost:', f1_score)\n",
    "    print('ROC AUC CatBoost:', roc_auc)\n",
    "    print('\\n')\n",
    "else:\n",
    "    mean_absolute_error = sklearn.metrics.mean_absolute_error(y_test, np.round(preds_grande))\n",
    "    r2_score = sklearn.metrics.r2_score(y_test, np.round(preds_grande))\n",
    "\n",
    "    print('MAE GRANDE:', mean_absolute_error)\n",
    "    print('R2 Score GRANDE:', r2_score)\n",
    "    print('\\n')\n",
    "\n",
    "    mean_absolute_error = sklearn.metrics.mean_absolute_error(y_test, np.round(preds_xgb))\n",
    "    r2_score = sklearn.metrics.r2_score(y_test, np.round(preds_xgb))\n",
    "\n",
    "    print('MAE XGB:', mean_absolute_error)\n",
    "    print('R2 Score XGB:', r2_score)\n",
    "    print('\\n')\n",
    "\n",
    "    mean_absolute_error = sklearn.metrics.mean_absolute_error(y_test, np.round(preds_catboost))\n",
    "    r2_score = sklearn.metrics.r2_score(y_test, np.round(preds_catboost))\n",
    "\n",
    "    print('MAE CatBoost:', mean_absolute_error)\n",
    "    print('R2 Score CatBoost:', r2_score)\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ef0e2bc0-9b43-4e5c-9f55-d1dae0de8930",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-15T11:57:52.454908Z",
     "iopub.status.busy": "2024-03-15T11:57:52.454754Z",
     "iopub.status.idle": "2024-03-15T11:57:52.834130Z",
     "shell.execute_reply": "2024-03-15T11:57:52.833314Z",
     "shell.execute_reply.started": "2024-03-15T11:57:52.454892Z"
    }
   },
   "outputs": [],
   "source": [
    "from numba import cuda \n",
    "device = cuda.get_current_device()\n",
    "device.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7ef884c-1269-4e0b-8ca7-7ea74c6e380c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}