GRANDE_minimal_example_with_comparison_REG.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "19407948-2e33-4576-ac7a-1c76819b421a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:21:09.111188Z",
     "iopub.status.busy": "2024-03-26T09:21:09.110920Z",
     "iopub.status.idle": "2024-03-26T09:21:09.115226Z",
     "shell.execute_reply": "2024-03-26T09:21:09.114478Z",
     "shell.execute_reply.started": "2024-03-26T09:21:09.111155Z"
    }
   },
   "outputs": [],
   "source": [
    "#!pip install catboost\n",
    "#!pip install xgboost\n",
    "#!pip install openml\n",
    "#!pip install numba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bf584ce8-8849-4a53-8ce9-2de8f9dd752d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:21:09.117382Z",
     "iopub.status.busy": "2024-03-26T09:21:09.117109Z",
     "iopub.status.idle": "2024-03-26T09:21:09.127281Z",
     "shell.execute_reply": "2024-03-26T09:21:09.126523Z",
     "shell.execute_reply.started": "2024-03-26T09:21:09.117351Z"
    }
   },
   "outputs": [],
   "source": [
    "#specify GPU to use\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "52b07238-f972-45df-86ce-a11cc11d72e3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:21:09.128755Z",
     "iopub.status.busy": "2024-03-26T09:21:09.128296Z",
     "iopub.status.idle": "2024-03-26T09:21:13.401197Z",
     "shell.execute_reply": "2024-03-26T09:21:13.399805Z",
     "shell.execute_reply.started": "2024-03-26T09:21:09.128722Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size: 6872\n",
      "Validation set size: 1718\n",
      "Test set size: 2148\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>productId</th>\n",
       "      <th>machineId</th>\n",
       "      <th>temp</th>\n",
       "      <th>weather_condition_id</th>\n",
       "      <th>isholiday</th>\n",
       "      <th>daysoff</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>week_day</th>\n",
       "      <th>avail0</th>\n",
       "      <th>avail1</th>\n",
       "      <th>avail2</th>\n",
       "      <th>stdv</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9369</th>\n",
       "      <td>0.548387</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.559935</td>\n",
       "      <td>0.994048</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.50</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.67</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6363</th>\n",
       "      <td>0.806452</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.655238</td>\n",
       "      <td>0.992063</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.83</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9859</th>\n",
       "      <td>0.064516</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.470842</td>\n",
       "      <td>0.992063</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2307</th>\n",
       "      <td>0.548387</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.148488</td>\n",
       "      <td>0.992063</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.83</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2139</th>\n",
       "      <td>0.967742</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.337473</td>\n",
       "      <td>0.396825</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.433333</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.50</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      productId  machineId      temp  weather_condition_id  isholiday  \\\n",
       "9369   0.548387        0.0  0.559935              0.994048          0   \n",
       "6363   0.806452        0.5  0.655238              0.992063          0   \n",
       "9859   0.064516        1.0  0.470842              0.992063          0   \n",
       "2307   0.548387        1.0  0.148488              0.992063          0   \n",
       "2139   0.967742        1.0  0.337473              0.396825          0   \n",
       "\n",
       "      daysoff  year     month       day  week_day  avail0  avail1  avail2  \\\n",
       "9369      0.0     1  0.727273  0.500000  0.000000    0.50    1.00    0.67   \n",
       "6363      0.0     1  0.545455  0.033333  0.166667    1.00    0.83    1.00   \n",
       "9859      0.0     1  0.727273  0.933333  1.000000    1.00    1.00    1.00   \n",
       "2307      0.0     1  0.181818  0.666667  0.500000    0.83    0.50    0.83   \n",
       "2139      0.0     1  0.181818  0.433333  0.500000    0.50    1.00    1.00   \n",
       "\n",
       "      stdv  \n",
       "9369     1  \n",
       "6363     1  \n",
       "9859     0  \n",
       "2307     2  \n",
       "2139     1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import openml\n",
    "import category_encoders as ce\n",
    "import numpy as np\n",
    "import sklearn\n",
    "\n",
    "# Load dataset_sales dataset\n",
    "dataset = openml.datasets.get_dataset(42183, download_data=True, download_qualities=True, download_features_meta_data=True)\n",
    "X, y, categorical_indicator, attribute_names = dataset.get_data(target=dataset.default_target_attribute)\n",
    "categorical_feature_indices = [idx for idx, idx_bool in enumerate(categorical_indicator) if idx_bool]\n",
    "\n",
    "X_temp, X_test, y_temp, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_temp, y_temp, test_size=0.2, random_state=42)\n",
    "\n",
    "print(\"Training set size:\", len(X_train))\n",
    "print(\"Validation set size:\", len(X_valid))\n",
    "print(\"Test set size:\", len(X_test))\n",
    "\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bf0a93c2-e9a4-4fa5-a7f6-44689dd07d37",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:21:13.403785Z",
     "iopub.status.busy": "2024-03-26T09:21:13.402909Z",
     "iopub.status.idle": "2024-03-26T09:24:36.659422Z",
     "shell.execute_reply": "2024-03-26T09:24:36.656589Z",
     "shell.execute_reply.started": "2024-03-26T09:21:13.403732Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-26 10:21:15.016663: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-03-26 10:21:19.911137: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "2024-03-26 10:22:00.787376: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:47] Overriding orig_value setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
      "2024-03-26 10:22:00.787852: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1928] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 9546 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:02:00.0, compute capability: 7.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1711444936.282159 3119067 service.cc:145] XLA service 0x7f691c003790 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "I0000 00:00:1711444936.282354 3119067 service.cc:153]   StreamExecutor device (0): NVIDIA GeForce RTX 2080 Ti, Compute Capability 7.5\n",
      "2024-03-26 10:22:16.688349: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-03-26 10:22:18.358735: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:465] Loaded cuDNN version 8902\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m 18/108\u001b[0m \u001b[32m━━━\u001b[0m\u001b[37m━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 10ms/step - loss: 0.8053"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0000 00:00:1711444940.390766 3119067 device_compiler.h:188] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m23s\u001b[0m 86ms/step - loss: 0.8023 - val_loss: 0.7273\n",
      "Epoch 2/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.5520 - val_loss: 0.5437\n",
      "Epoch 3/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 13ms/step - loss: 0.4041 - val_loss: 0.4775\n",
      "Epoch 4/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.3338 - val_loss: 0.4243\n",
      "Epoch 5/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 9ms/step - loss: 0.2760 - val_loss: 0.3835\n",
      "Epoch 6/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.2504 - val_loss: 0.3496\n",
      "Epoch 7/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.2260 - val_loss: 0.3140\n",
      "Epoch 8/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 13ms/step - loss: 0.2095 - val_loss: 0.3045\n",
      "Epoch 9/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 13ms/step - loss: 0.2025 - val_loss: 0.2849\n",
      "Epoch 10/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.1971 - val_loss: 0.2837\n",
      "Epoch 11/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 13ms/step - loss: 0.1884 - val_loss: 0.2591\n",
      "Epoch 12/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 12ms/step - loss: 0.1800 - val_loss: 0.2504\n",
      "Epoch 13/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 13ms/step - loss: 0.1763 - val_loss: 0.2448\n",
      "Epoch 14/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1740 - val_loss: 0.2410\n",
      "Epoch 15/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1701 - val_loss: 0.2369\n",
      "Epoch 16/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1676 - val_loss: 0.2439\n",
      "Epoch 17/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.1636 - val_loss: 0.2578\n",
      "Epoch 18/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1642 - val_loss: 0.2321\n",
      "Epoch 19/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.1611 - val_loss: 0.2272\n",
      "Epoch 20/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1571 - val_loss: 0.2235\n",
      "Epoch 21/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1568 - val_loss: 0.2224\n",
      "Epoch 22/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1510 - val_loss: 0.2175\n",
      "Epoch 23/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1514 - val_loss: 0.2223\n",
      "Epoch 24/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1476 - val_loss: 0.2214\n",
      "Epoch 25/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1518 - val_loss: 0.2196\n",
      "Epoch 26/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1473 - val_loss: 0.2203\n",
      "Epoch 27/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1443 - val_loss: 0.2141\n",
      "Epoch 28/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1454 - val_loss: 0.2180\n",
      "Epoch 29/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 13ms/step - loss: 0.1409 - val_loss: 0.2137\n",
      "Epoch 30/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 11ms/step - loss: 0.1356 - val_loss: 0.2063\n",
      "Epoch 31/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1340 - val_loss: 0.2058\n",
      "Epoch 32/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1331 - val_loss: 0.2076\n",
      "Epoch 33/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 13ms/step - loss: 0.1303 - val_loss: 0.2024\n",
      "Epoch 34/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1296 - val_loss: 0.2062\n",
      "Epoch 35/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1271 - val_loss: 0.2058\n",
      "Epoch 36/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1258 - val_loss: 0.2019\n",
      "Epoch 37/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1234 - val_loss: 0.2015\n",
      "Epoch 38/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1264 - val_loss: 0.2020\n",
      "Epoch 39/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1242 - val_loss: 0.2000\n",
      "Epoch 40/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1214 - val_loss: 0.2004\n",
      "Epoch 41/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1205 - val_loss: 0.1967\n",
      "Epoch 42/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1184 - val_loss: 0.1963\n",
      "Epoch 43/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.1157 - val_loss: 0.1930\n",
      "Epoch 44/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1192 - val_loss: 0.1923\n",
      "Epoch 45/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1152 - val_loss: 0.1962\n",
      "Epoch 46/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1124 - val_loss: 0.1955\n",
      "Epoch 47/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1139 - val_loss: 0.1958\n",
      "Epoch 48/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1123 - val_loss: 0.1967\n",
      "Epoch 49/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.1156 - val_loss: 0.1977\n",
      "Epoch 50/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1068 - val_loss: 0.1921\n",
      "Epoch 51/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1065 - val_loss: 0.1912\n",
      "Epoch 52/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1081 - val_loss: 0.1996\n",
      "Epoch 53/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1060 - val_loss: 0.1998\n",
      "Epoch 54/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1073 - val_loss: 0.2008\n",
      "Epoch 55/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 14ms/step - loss: 0.1046 - val_loss: 0.1973\n",
      "Epoch 56/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1100 - val_loss: 0.1962\n",
      "Epoch 57/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.1073 - val_loss: 0.1993\n",
      "Epoch 58/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1011 - val_loss: 0.1961\n",
      "Epoch 59/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1017 - val_loss: 0.1949\n",
      "Epoch 60/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1012 - val_loss: 0.1947\n",
      "Epoch 61/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.1035 - val_loss: 0.1920\n",
      "Epoch 62/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0967 - val_loss: 0.1869\n",
      "Epoch 63/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0942 - val_loss: 0.1897\n",
      "Epoch 64/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0960 - val_loss: 0.1946\n",
      "Epoch 65/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0940 - val_loss: 0.1901\n",
      "Epoch 66/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0952 - val_loss: 0.1912\n",
      "Epoch 67/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0951 - val_loss: 0.1891\n",
      "Epoch 68/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0950 - val_loss: 0.1915\n",
      "Epoch 69/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0910 - val_loss: 0.1960\n",
      "Epoch 70/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0914 - val_loss: 0.1857\n",
      "Epoch 71/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0934 - val_loss: 0.1961\n",
      "Epoch 72/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0883 - val_loss: 0.1869\n",
      "Epoch 73/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0887 - val_loss: 0.1939\n",
      "Epoch 74/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0905 - val_loss: 0.1885\n",
      "Epoch 75/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0903 - val_loss: 0.1904\n",
      "Epoch 76/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0888 - val_loss: 0.1963\n",
      "Epoch 77/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0900 - val_loss: 0.1894\n",
      "Epoch 78/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0890 - val_loss: 0.1962\n",
      "Epoch 79/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.0853 - val_loss: 0.1870\n",
      "Epoch 80/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.0865 - val_loss: 0.1871\n",
      "Epoch 81/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.0839 - val_loss: 0.1884\n",
      "Epoch 82/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 12ms/step - loss: 0.0849 - val_loss: 0.1854\n",
      "Epoch 83/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 13ms/step - loss: 0.0811 - val_loss: 0.1866\n",
      "Epoch 84/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 13ms/step - loss: 0.0805 - val_loss: 0.1874\n",
      "Epoch 85/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0786 - val_loss: 0.1852\n",
      "Epoch 86/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 13ms/step - loss: 0.0787 - val_loss: 0.1902\n",
      "Epoch 87/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0863 - val_loss: 0.2088\n",
      "Epoch 88/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0948 - val_loss: 0.1931\n",
      "Epoch 89/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0914 - val_loss: 0.2112\n",
      "Epoch 90/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0913 - val_loss: 0.2016\n",
      "Epoch 91/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0850 - val_loss: 0.1898\n",
      "Epoch 92/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0804 - val_loss: 0.1878\n",
      "Epoch 93/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0806 - val_loss: 0.1938\n",
      "Epoch 94/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 10ms/step - loss: 0.0789 - val_loss: 0.1975\n",
      "Epoch 95/1000\n",
      "\u001b[1m108/108\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step - loss: 0.0803 - val_loss: 0.1864\n"
     ]
    }
   ],
   "source": [
    "from GRANDE import GRANDE\n",
    "\n",
    "params = {\n",
    "        'depth': 5, # tree depth\n",
    "        'n_estimators': 2048, # number of estimators / trees\n",
    "\n",
    "        'learning_rate_weights': 0.005, # learning rate for leaf weights\n",
    "        'learning_rate_index': 0.01, # learning rate for split indices\n",
    "        'learning_rate_values': 0.01, # learning rate for split values\n",
    "        'learning_rate_leaf': 0.01, # learning rate for leafs (logits)\n",
    "\n",
    "        'optimizer': 'adam', # optimizer\n",
    "        'cosine_decay_steps': 0, # decay steps for lr schedule (CosineDecayRestarts)\n",
    "\n",
    "        'loss': 'mse', # loss function (default 'crossentropy' for binary & multi-class classification and 'mse' for regression)\n",
    "        'focal_loss': False, # use focal loss {True, False}\n",
    "        'temperature': 0.0, # temperature for stochastic re-weighted GD (0.0, 1.0)\n",
    "\n",
    "        'from_logits': True, # use logits for weighting {True, False}\n",
    "        'use_class_weights': True, # use class weights for training {True, False}\n",
    "\n",
    "        'dropout': 0.0, # dropout rate (here, dropout randomly disables individual estimators of the ensemble during training)\n",
    "\n",
    "        'selected_variables': 0.8, # feature subset percentage (0.0, 1.0)\n",
    "        'data_subset_fraction': 1.0, # data subset percentage (0.0, 1.0)\n",
    "}\n",
    "\n",
    "args = {\n",
    "    'epochs': 1_000, # number of epochs for training\n",
    "    'early_stopping_epochs': 25, # patience for early stopping (best weights are restored)\n",
    "    'batch_size': 64,  # batch size for training\n",
    "\n",
    "    'cat_idx': categorical_feature_indices, # put list of categorical indices\n",
    "    'objective': 'regression', # objective / task {'binary', 'classification', 'regression'}\n",
    "    \n",
    "    'random_seed': 42,\n",
    "    'verbose': 1,       \n",
    "}\n",
    "\n",
    "model_grande = GRANDE(params=params, args=args)\n",
    "\n",
    "model_grande.fit(X_train=X_train,\n",
    "          y_train=y_train,\n",
    "          X_val=X_valid,\n",
    "          y_val=y_valid)\n",
    "\n",
    "preds_grande = model_grande.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b251df9d-67f0-4dd1-a0cc-379621e33fad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:24:36.664964Z",
     "iopub.status.busy": "2024-03-26T09:24:36.662970Z",
     "iopub.status.idle": "2024-03-26T09:24:36.675604Z",
     "shell.execute_reply": "2024-03-26T09:24:36.673718Z",
     "shell.execute_reply.started": "2024-03-26T09:24:36.664845Z"
    }
   },
   "outputs": [],
   "source": [
    "def calculate_sample_weights(y_data):\n",
    "    class_weights = sklearn.utils.class_weight.compute_class_weight(class_weight = 'balanced', classes = np.unique(y_data), y = y_data)\n",
    "    sample_weights = sklearn.utils.class_weight.compute_sample_weight(class_weight = 'balanced', y =y_data)\n",
    "    return sample_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f7e55785-bf56-4aad-9825-7c5737be5ab7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:24:36.678684Z",
     "iopub.status.busy": "2024-03-26T09:24:36.677624Z",
     "iopub.status.idle": "2024-03-26T09:24:36.691510Z",
     "shell.execute_reply": "2024-03-26T09:24:36.689588Z",
     "shell.execute_reply.started": "2024-03-26T09:24:36.678612Z"
    }
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    y_train = y_train.values.codes.astype(np.float64)\n",
    "    y_valid = y_valid.values.codes.astype(np.float64)\n",
    "    y_test = y_test.values.codes.astype(np.float64)\n",
    "except:\n",
    "    y_train = y_train.values.astype(np.float64)\n",
    "    y_valid = y_valid.values.astype(np.float64)\n",
    "    y_test = y_test.values.astype(np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e9622a72-43ca-4e78-b2eb-bb27bdb37992",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:24:36.694617Z",
     "iopub.status.busy": "2024-03-26T09:24:36.693631Z",
     "iopub.status.idle": "2024-03-26T09:24:36.706987Z",
     "shell.execute_reply": "2024-03-26T09:24:36.705043Z",
     "shell.execute_reply.started": "2024-03-26T09:24:36.694555Z"
    }
   },
   "outputs": [],
   "source": [
    "binary_indices = []\n",
    "low_cardinality_indices = []\n",
    "high_cardinality_indices = []\n",
    "num_columns = []\n",
    "for column_index, column in enumerate(X_train.columns):\n",
    "    if column_index in categorical_feature_indices:\n",
    "        if len(X_train.iloc[:,column_index].unique()) <= 2:\n",
    "            binary_indices.append(column)\n",
    "        elif len(X_train.iloc[:,column_index].unique()) < 5:\n",
    "            low_cardinality_indices.append(column)\n",
    "        else:\n",
    "            high_cardinality_indices.append(column)\n",
    "    else:\n",
    "        num_columns.append(column)     \n",
    "cat_columns = [col for col in X_train.columns if col not in num_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d11cf7a3-9247-49a7-a75d-9766013c8744",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:24:36.713301Z",
     "iopub.status.busy": "2024-03-26T09:24:36.712469Z",
     "iopub.status.idle": "2024-03-26T09:24:36.775425Z",
     "shell.execute_reply": "2024-03-26T09:24:36.773422Z",
     "shell.execute_reply.started": "2024-03-26T09:24:36.713245Z"
    }
   },
   "outputs": [],
   "source": [
    "if len(num_columns) > 0:\n",
    "    mean_train_num = X_train[num_columns].mean(axis=0).iloc[0]\n",
    "    X_train[num_columns] = X_train[num_columns].fillna(mean_train_num)\n",
    "    X_valid[num_columns] = X_valid[num_columns].fillna(mean_train_num)\n",
    "    X_test[num_columns] = X_test[num_columns].fillna(mean_train_num)\n",
    "if len(cat_columns) > 0:\n",
    "    mode_train_cat = X_train[cat_columns].mode(axis=0).iloc[0]\n",
    "    X_train[cat_columns] = X_train[cat_columns].fillna(mode_train_cat)\n",
    "    X_valid[cat_columns] = X_valid[cat_columns].fillna(mode_train_cat)\n",
    "    X_test[cat_columns] = X_test[cat_columns].fillna(mode_train_cat)\n",
    "\n",
    "X_train_raw = X_train.copy()\n",
    "X_valid_raw = X_valid.copy()\n",
    "X_test_raw = X_test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b5d5f00f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:24:36.778910Z",
     "iopub.status.busy": "2024-03-26T09:24:36.777982Z",
     "iopub.status.idle": "2024-03-26T09:24:36.818054Z",
     "shell.execute_reply": "2024-03-26T09:24:36.815708Z",
     "shell.execute_reply.started": "2024-03-26T09:24:36.778831Z"
    }
   },
   "outputs": [],
   "source": [
    "encoder_ordinal = ce.OrdinalEncoder(cols=binary_indices)\n",
    "encoder_ordinal.fit(X_train)\n",
    "X_train = encoder_ordinal.transform(X_train)\n",
    "X_valid = encoder_ordinal.transform(X_valid)  \n",
    "X_test = encoder_ordinal.transform(X_test)       \n",
    "\n",
    "encoder = ce.LeaveOneOutEncoder(cols=high_cardinality_indices)\n",
    "encoder.fit(X_train, y_train)\n",
    "X_train = encoder.transform(X_train)\n",
    "X_valid = encoder.transform(X_valid)\n",
    "X_test = encoder.transform(X_test)\n",
    "\n",
    "encoder = ce.OneHotEncoder(cols=low_cardinality_indices)\n",
    "encoder.fit(X_train)\n",
    "X_train = encoder.transform(X_train)\n",
    "X_valid = encoder.transform(X_valid)\n",
    "X_test = encoder.transform(X_test)\n",
    "\n",
    "X_train = X_train.astype(np.float32)\n",
    "X_valid = X_valid.astype(np.float32)\n",
    "X_test = X_test.astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "361f7d9f-7e58-4bc1-8855-153b9e972742",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:24:36.821889Z",
     "iopub.status.busy": "2024-03-26T09:24:36.820857Z",
     "iopub.status.idle": "2024-03-26T09:24:51.845725Z",
     "shell.execute_reply": "2024-03-26T09:24:51.843604Z",
     "shell.execute_reply.started": "2024-03-26T09:24:36.821827Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:21.32764\n",
      "[1]\tvalidation_0-rmse:17.83969\n",
      "[2]\tvalidation_0-rmse:14.79749\n",
      "[3]\tvalidation_0-rmse:13.09891\n",
      "[4]\tvalidation_0-rmse:11.70010\n",
      "[5]\tvalidation_0-rmse:11.22561\n",
      "[6]\tvalidation_0-rmse:10.70027\n",
      "[7]\tvalidation_0-rmse:10.36325\n",
      "[8]\tvalidation_0-rmse:10.00313\n",
      "[9]\tvalidation_0-rmse:9.87188\n",
      "[10]\tvalidation_0-rmse:9.81904\n",
      "[11]\tvalidation_0-rmse:9.75513\n",
      "[12]\tvalidation_0-rmse:9.68747\n",
      "[13]\tvalidation_0-rmse:9.63047\n",
      "[14]\tvalidation_0-rmse:9.62859\n",
      "[15]\tvalidation_0-rmse:9.61208\n",
      "[16]\tvalidation_0-rmse:9.58495\n",
      "[17]\tvalidation_0-rmse:9.63782\n",
      "[18]\tvalidation_0-rmse:9.61342\n",
      "[19]\tvalidation_0-rmse:9.72494\n",
      "[20]\tvalidation_0-rmse:9.68926\n",
      "[21]\tvalidation_0-rmse:9.66877\n",
      "[22]\tvalidation_0-rmse:9.65851\n",
      "[23]\tvalidation_0-rmse:9.63861\n",
      "[24]\tvalidation_0-rmse:9.63711\n",
      "[25]\tvalidation_0-rmse:9.62849\n",
      "[26]\tvalidation_0-rmse:9.61850\n",
      "[27]\tvalidation_0-rmse:9.61490\n",
      "[28]\tvalidation_0-rmse:9.62154\n",
      "[29]\tvalidation_0-rmse:9.61162\n",
      "[30]\tvalidation_0-rmse:9.60180\n",
      "[31]\tvalidation_0-rmse:9.60386\n",
      "[32]\tvalidation_0-rmse:9.62915\n",
      "[33]\tvalidation_0-rmse:9.62021\n",
      "[34]\tvalidation_0-rmse:9.61674\n",
      "[35]\tvalidation_0-rmse:9.61412\n",
      "[36]\tvalidation_0-rmse:9.64740\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBRegressor\n",
    "model_xgb = XGBRegressor(n_estimators=1000, early_stopping_rounds=20)\n",
    "model_xgb.fit(X_train, \n",
    "                y_train, \n",
    "                sample_weight=calculate_sample_weights(y_train), \n",
    "                eval_set=[(X_valid, y_valid)], \n",
    "                sample_weight_eval_set=[calculate_sample_weights(y_valid)]\n",
    "             )\n",
    "if args['objective'] == 'regression':\n",
    "    preds_xgb = model_xgb.predict(X_test)\n",
    "else:\n",
    "    preds_xgb = model_xgb.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6b23c67e-cdde-40c2-842a-61383e1a62d0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:24:51.850429Z",
     "iopub.status.busy": "2024-03-26T09:24:51.849000Z",
     "iopub.status.idle": "2024-03-26T09:24:55.085689Z",
     "shell.execute_reply": "2024-03-26T09:24:55.084054Z",
     "shell.execute_reply.started": "2024-03-26T09:24:51.850347Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.068916\n",
      "0:\tlearn: 13.8938300\ttest: 14.8375117\tbest: 14.8375117 (0)\ttotal: 71.1ms\tremaining: 1m 11s\n",
      "1:\tlearn: 13.4868750\ttest: 14.6356384\tbest: 14.6356384 (1)\ttotal: 81.8ms\tremaining: 40.8s\n",
      "2:\tlearn: 13.1038555\ttest: 14.3930520\tbest: 14.3930520 (2)\ttotal: 88.2ms\tremaining: 29.3s\n",
      "3:\tlearn: 12.7778368\ttest: 14.2964681\tbest: 14.2964681 (3)\ttotal: 92.8ms\tremaining: 23.1s\n",
      "4:\tlearn: 12.4106060\ttest: 14.1245314\tbest: 14.1245314 (4)\ttotal: 99.4ms\tremaining: 19.8s\n",
      "5:\tlearn: 12.1370113\ttest: 13.9665072\tbest: 13.9665072 (5)\ttotal: 105ms\tremaining: 17.4s\n",
      "6:\tlearn: 11.8227111\ttest: 13.8241105\tbest: 13.8241105 (6)\ttotal: 109ms\tremaining: 15.5s\n",
      "7:\tlearn: 11.5206139\ttest: 13.7326919\tbest: 13.7326919 (7)\ttotal: 115ms\tremaining: 14.3s\n",
      "8:\tlearn: 11.1639396\ttest: 13.5592696\tbest: 13.5592696 (8)\ttotal: 122ms\tremaining: 13.4s\n",
      "9:\tlearn: 10.9145692\ttest: 13.4335573\tbest: 13.4335573 (9)\ttotal: 127ms\tremaining: 12.6s\n",
      "10:\tlearn: 10.5669160\ttest: 13.2766710\tbest: 13.2766710 (10)\ttotal: 132ms\tremaining: 11.8s\n",
      "11:\tlearn: 10.3055486\ttest: 13.0793165\tbest: 13.0793165 (11)\ttotal: 143ms\tremaining: 11.7s\n",
      "12:\tlearn: 10.0138241\ttest: 13.0023147\tbest: 13.0023147 (12)\ttotal: 152ms\tremaining: 11.6s\n",
      "13:\tlearn: 9.7793630\ttest: 12.9597435\tbest: 12.9597435 (13)\ttotal: 157ms\tremaining: 11s\n",
      "14:\tlearn: 9.5695990\ttest: 12.9224360\tbest: 12.9224360 (14)\ttotal: 161ms\tremaining: 10.6s\n",
      "15:\tlearn: 9.4139876\ttest: 12.8392991\tbest: 12.8392991 (15)\ttotal: 167ms\tremaining: 10.3s\n",
      "16:\tlearn: 9.2171709\ttest: 12.6774271\tbest: 12.6774271 (16)\ttotal: 173ms\tremaining: 9.98s\n",
      "17:\tlearn: 9.0024598\ttest: 12.5772392\tbest: 12.5772392 (17)\ttotal: 178ms\tremaining: 9.73s\n",
      "18:\tlearn: 8.8460844\ttest: 12.5184913\tbest: 12.5184913 (18)\ttotal: 184ms\tremaining: 9.49s\n",
      "19:\tlearn: 8.5638650\ttest: 12.3691460\tbest: 12.3691460 (19)\ttotal: 189ms\tremaining: 9.27s\n",
      "20:\tlearn: 8.4108983\ttest: 12.3129850\tbest: 12.3129850 (20)\ttotal: 194ms\tremaining: 9.05s\n",
      "21:\tlearn: 8.2579178\ttest: 12.1967008\tbest: 12.1967008 (21)\ttotal: 199ms\tremaining: 8.83s\n",
      "22:\tlearn: 8.1480454\ttest: 12.1704200\tbest: 12.1704200 (22)\ttotal: 204ms\tremaining: 8.66s\n",
      "23:\tlearn: 7.9784102\ttest: 12.1065220\tbest: 12.1065220 (23)\ttotal: 209ms\tremaining: 8.48s\n",
      "24:\tlearn: 7.8412956\ttest: 12.0758575\tbest: 12.0758575 (24)\ttotal: 214ms\tremaining: 8.34s\n",
      "25:\tlearn: 7.7778973\ttest: 12.0335977\tbest: 12.0335977 (25)\ttotal: 220ms\tremaining: 8.24s\n",
      "26:\tlearn: 7.6862360\ttest: 11.9592447\tbest: 11.9592447 (26)\ttotal: 225ms\tremaining: 8.12s\n",
      "27:\tlearn: 7.5657389\ttest: 11.9131638\tbest: 11.9131638 (27)\ttotal: 231ms\tremaining: 8s\n",
      "28:\tlearn: 7.4850902\ttest: 11.8524156\tbest: 11.8524156 (28)\ttotal: 249ms\tremaining: 8.34s\n",
      "29:\tlearn: 7.4093194\ttest: 11.8463993\tbest: 11.8463993 (29)\ttotal: 255ms\tremaining: 8.25s\n",
      "30:\tlearn: 7.3161981\ttest: 11.8506723\tbest: 11.8463993 (29)\ttotal: 260ms\tremaining: 8.12s\n",
      "31:\tlearn: 7.2749246\ttest: 11.8312969\tbest: 11.8312969 (31)\ttotal: 263ms\tremaining: 7.94s\n",
      "32:\tlearn: 7.0900225\ttest: 11.7128642\tbest: 11.7128642 (32)\ttotal: 268ms\tremaining: 7.85s\n",
      "33:\tlearn: 7.0368133\ttest: 11.6748090\tbest: 11.6748090 (33)\ttotal: 292ms\tremaining: 8.29s\n",
      "34:\tlearn: 6.9268045\ttest: 11.5986759\tbest: 11.5986759 (34)\ttotal: 310ms\tremaining: 8.55s\n",
      "35:\tlearn: 6.8788960\ttest: 11.5981120\tbest: 11.5981120 (35)\ttotal: 319ms\tremaining: 8.53s\n",
      "36:\tlearn: 6.8305965\ttest: 11.5619627\tbest: 11.5619627 (36)\ttotal: 328ms\tremaining: 8.55s\n",
      "37:\tlearn: 6.7691820\ttest: 11.5231109\tbest: 11.5231109 (37)\ttotal: 337ms\tremaining: 8.53s\n",
      "38:\tlearn: 6.7128611\ttest: 11.5244755\tbest: 11.5231109 (37)\ttotal: 348ms\tremaining: 8.57s\n",
      "39:\tlearn: 6.6509460\ttest: 11.4892460\tbest: 11.4892460 (39)\ttotal: 354ms\tremaining: 8.5s\n",
      "40:\tlearn: 6.5314453\ttest: 11.3833452\tbest: 11.3833452 (40)\ttotal: 357ms\tremaining: 8.35s\n",
      "41:\tlearn: 6.4708076\ttest: 11.3659828\tbest: 11.3659828 (41)\ttotal: 389ms\tremaining: 8.87s\n",
      "42:\tlearn: 6.3466572\ttest: 11.2732814\tbest: 11.2732814 (42)\ttotal: 398ms\tremaining: 8.85s\n",
      "43:\tlearn: 6.3061661\ttest: 11.2481403\tbest: 11.2481403 (43)\ttotal: 402ms\tremaining: 8.72s\n",
      "44:\tlearn: 6.2210914\ttest: 11.2195234\tbest: 11.2195234 (44)\ttotal: 421ms\tremaining: 8.93s\n",
      "45:\tlearn: 6.1514114\ttest: 11.1864188\tbest: 11.1864188 (45)\ttotal: 424ms\tremaining: 8.79s\n",
      "46:\tlearn: 6.1146126\ttest: 11.1661226\tbest: 11.1661226 (46)\ttotal: 428ms\tremaining: 8.67s\n",
      "47:\tlearn: 6.0859629\ttest: 11.1265713\tbest: 11.1265713 (47)\ttotal: 432ms\tremaining: 8.57s\n",
      "48:\tlearn: 6.0582836\ttest: 11.0881508\tbest: 11.0881508 (48)\ttotal: 435ms\tremaining: 8.44s\n",
      "49:\tlearn: 6.0290538\ttest: 11.0594484\tbest: 11.0594484 (49)\ttotal: 441ms\tremaining: 8.38s\n",
      "50:\tlearn: 5.9433058\ttest: 11.0144840\tbest: 11.0144840 (50)\ttotal: 444ms\tremaining: 8.26s\n",
      "51:\tlearn: 5.8702561\ttest: 10.9919526\tbest: 10.9919526 (51)\ttotal: 447ms\tremaining: 8.15s\n",
      "52:\tlearn: 5.7944646\ttest: 10.9534050\tbest: 10.9534050 (52)\ttotal: 478ms\tremaining: 8.54s\n",
      "53:\tlearn: 5.7558944\ttest: 10.9085638\tbest: 10.9085638 (53)\ttotal: 481ms\tremaining: 8.43s\n",
      "54:\tlearn: 5.7263651\ttest: 10.8988091\tbest: 10.8988091 (54)\ttotal: 485ms\tremaining: 8.33s\n",
      "55:\tlearn: 5.7038093\ttest: 10.8990935\tbest: 10.8988091 (54)\ttotal: 488ms\tremaining: 8.22s\n",
      "56:\tlearn: 5.6755603\ttest: 10.9116728\tbest: 10.8988091 (54)\ttotal: 491ms\tremaining: 8.13s\n",
      "57:\tlearn: 5.6547085\ttest: 10.9067827\tbest: 10.8988091 (54)\ttotal: 494ms\tremaining: 8.02s\n",
      "58:\tlearn: 5.6241778\ttest: 10.8960742\tbest: 10.8960742 (58)\ttotal: 497ms\tremaining: 7.93s\n",
      "59:\tlearn: 5.5896009\ttest: 10.8983642\tbest: 10.8960742 (58)\ttotal: 508ms\tremaining: 7.96s\n",
      "60:\tlearn: 5.5620180\ttest: 10.8854145\tbest: 10.8854145 (60)\ttotal: 515ms\tremaining: 7.93s\n",
      "61:\tlearn: 5.5056004\ttest: 10.8695415\tbest: 10.8695415 (61)\ttotal: 518ms\tremaining: 7.84s\n",
      "62:\tlearn: 5.4820359\ttest: 10.8433756\tbest: 10.8433756 (62)\ttotal: 521ms\tremaining: 7.75s\n",
      "63:\tlearn: 5.4727590\ttest: 10.8481216\tbest: 10.8433756 (62)\ttotal: 524ms\tremaining: 7.67s\n",
      "64:\tlearn: 5.4491808\ttest: 10.8019130\tbest: 10.8019130 (64)\ttotal: 529ms\tremaining: 7.61s\n",
      "65:\tlearn: 5.3950787\ttest: 10.7573771\tbest: 10.7573771 (65)\ttotal: 546ms\tremaining: 7.73s\n",
      "66:\tlearn: 5.3733232\ttest: 10.7492025\tbest: 10.7492025 (66)\ttotal: 549ms\tremaining: 7.64s\n",
      "67:\tlearn: 5.3713748\ttest: 10.7362127\tbest: 10.7362127 (67)\ttotal: 550ms\tremaining: 7.54s\n",
      "68:\tlearn: 5.3518992\ttest: 10.7207447\tbest: 10.7207447 (68)\ttotal: 553ms\tremaining: 7.47s\n",
      "69:\tlearn: 5.3299856\ttest: 10.7154950\tbest: 10.7154950 (69)\ttotal: 556ms\tremaining: 7.39s\n",
      "70:\tlearn: 5.2968719\ttest: 10.7030753\tbest: 10.7030753 (70)\ttotal: 565ms\tremaining: 7.39s\n",
      "71:\tlearn: 5.2775978\ttest: 10.7056256\tbest: 10.7030753 (70)\ttotal: 568ms\tremaining: 7.32s\n",
      "72:\tlearn: 5.2437579\ttest: 10.6987003\tbest: 10.6987003 (72)\ttotal: 570ms\tremaining: 7.24s\n",
      "73:\tlearn: 5.2315422\ttest: 10.6868829\tbest: 10.6868829 (73)\ttotal: 573ms\tremaining: 7.17s\n",
      "74:\tlearn: 5.2040945\ttest: 10.6752032\tbest: 10.6752032 (74)\ttotal: 576ms\tremaining: 7.1s\n",
      "75:\tlearn: 5.1732748\ttest: 10.6878414\tbest: 10.6752032 (74)\ttotal: 578ms\tremaining: 7.03s\n",
      "76:\tlearn: 5.1291545\ttest: 10.6630847\tbest: 10.6630847 (76)\ttotal: 581ms\tremaining: 6.97s\n",
      "77:\tlearn: 5.0932932\ttest: 10.6449504\tbest: 10.6449504 (77)\ttotal: 585ms\tremaining: 6.91s\n",
      "78:\tlearn: 5.0748436\ttest: 10.6360003\tbest: 10.6360003 (78)\ttotal: 588ms\tremaining: 6.85s\n",
      "79:\tlearn: 5.0440008\ttest: 10.6594261\tbest: 10.6360003 (78)\ttotal: 590ms\tremaining: 6.79s\n",
      "80:\tlearn: 5.0022641\ttest: 10.6315532\tbest: 10.6315532 (80)\ttotal: 600ms\tremaining: 6.8s\n",
      "81:\tlearn: 4.9795589\ttest: 10.6438251\tbest: 10.6315532 (80)\ttotal: 603ms\tremaining: 6.75s\n",
      "82:\tlearn: 4.9417543\ttest: 10.5927153\tbest: 10.5927153 (82)\ttotal: 606ms\tremaining: 6.69s\n",
      "83:\tlearn: 4.9240675\ttest: 10.5800403\tbest: 10.5800403 (83)\ttotal: 609ms\tremaining: 6.65s\n",
      "84:\tlearn: 4.9049671\ttest: 10.5779781\tbest: 10.5779781 (84)\ttotal: 612ms\tremaining: 6.59s\n",
      "85:\tlearn: 4.8968185\ttest: 10.5749014\tbest: 10.5749014 (85)\ttotal: 616ms\tremaining: 6.54s\n",
      "86:\tlearn: 4.8780458\ttest: 10.5664935\tbest: 10.5664935 (86)\ttotal: 619ms\tremaining: 6.49s\n",
      "87:\tlearn: 4.8330547\ttest: 10.5426132\tbest: 10.5426132 (87)\ttotal: 621ms\tremaining: 6.44s\n",
      "88:\tlearn: 4.8126973\ttest: 10.5431343\tbest: 10.5426132 (87)\ttotal: 624ms\tremaining: 6.38s\n",
      "89:\tlearn: 4.7875526\ttest: 10.5622130\tbest: 10.5426132 (87)\ttotal: 626ms\tremaining: 6.33s\n",
      "90:\tlearn: 4.7750345\ttest: 10.5608806\tbest: 10.5426132 (87)\ttotal: 630ms\tremaining: 6.29s\n",
      "91:\tlearn: 4.7411837\ttest: 10.5926731\tbest: 10.5426132 (87)\ttotal: 634ms\tremaining: 6.25s\n",
      "92:\tlearn: 4.7220800\ttest: 10.5743113\tbest: 10.5426132 (87)\ttotal: 637ms\tremaining: 6.22s\n",
      "93:\tlearn: 4.7058799\ttest: 10.5938275\tbest: 10.5426132 (87)\ttotal: 641ms\tremaining: 6.17s\n",
      "94:\tlearn: 4.6878877\ttest: 10.6093218\tbest: 10.5426132 (87)\ttotal: 643ms\tremaining: 6.13s\n",
      "95:\tlearn: 4.6517554\ttest: 10.5923511\tbest: 10.5426132 (87)\ttotal: 646ms\tremaining: 6.08s\n",
      "96:\tlearn: 4.6336983\ttest: 10.5808334\tbest: 10.5426132 (87)\ttotal: 649ms\tremaining: 6.04s\n",
      "97:\tlearn: 4.6135302\ttest: 10.5898487\tbest: 10.5426132 (87)\ttotal: 653ms\tremaining: 6.01s\n",
      "98:\tlearn: 4.5718109\ttest: 10.5890648\tbest: 10.5426132 (87)\ttotal: 658ms\tremaining: 5.99s\n",
      "99:\tlearn: 4.5547861\ttest: 10.6061817\tbest: 10.5426132 (87)\ttotal: 661ms\tremaining: 5.95s\n",
      "100:\tlearn: 4.5291531\ttest: 10.5919770\tbest: 10.5426132 (87)\ttotal: 664ms\tremaining: 5.91s\n",
      "101:\tlearn: 4.5140038\ttest: 10.5855154\tbest: 10.5426132 (87)\ttotal: 667ms\tremaining: 5.87s\n",
      "102:\tlearn: 4.4877729\ttest: 10.5871234\tbest: 10.5426132 (87)\ttotal: 669ms\tremaining: 5.83s\n",
      "103:\tlearn: 4.4664321\ttest: 10.5772573\tbest: 10.5426132 (87)\ttotal: 673ms\tremaining: 5.8s\n",
      "104:\tlearn: 4.4384835\ttest: 10.5830266\tbest: 10.5426132 (87)\ttotal: 677ms\tremaining: 5.77s\n",
      "105:\tlearn: 4.4251894\ttest: 10.5946413\tbest: 10.5426132 (87)\ttotal: 680ms\tremaining: 5.73s\n",
      "106:\tlearn: 4.4114297\ttest: 10.5941718\tbest: 10.5426132 (87)\ttotal: 682ms\tremaining: 5.69s\n",
      "107:\tlearn: 4.3811357\ttest: 10.5770685\tbest: 10.5426132 (87)\ttotal: 686ms\tremaining: 5.67s\n",
      "Stopped by overfitting detector  (20 iterations wait)\n",
      "\n",
      "bestTest = 10.54261316\n",
      "bestIteration = 87\n",
      "\n",
      "Shrink model to first 88 iterations.\n"
     ]
    }
   ],
   "source": [
    "from catboost import CatBoostRegressor, Pool\n",
    "\n",
    "model_catboost = CatBoostRegressor(n_estimators=1000, \n",
    "                                    early_stopping_rounds=20)\n",
    "train_data = Pool(\n",
    "        data=X_train_raw,\n",
    "        label=y_train,\n",
    "        cat_features=categorical_feature_indices,\n",
    "        weight=calculate_sample_weights(y_train)\n",
    "    )\n",
    "\n",
    "eval_data = Pool(\n",
    "        data=X_valid_raw,\n",
    "        label=y_valid,\n",
    "        cat_features=categorical_feature_indices,\n",
    "        weight=calculate_sample_weights(y_valid),\n",
    "    )\n",
    "\n",
    "model_catboost.fit(X=train_data, \n",
    "                   eval_set=eval_data)\n",
    "\n",
    "if args['objective'] == 'regression':\n",
    "    preds_catboost = model_catboost.predict(X_test_raw)\n",
    "else:\n",
    "    preds_catboost = model_catboost.predict_proba(X_test_raw)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "21b9be01-ba30-4300-b3e0-f4fd43b55c87",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:24:55.088459Z",
     "iopub.status.busy": "2024-03-26T09:24:55.087850Z",
     "iopub.status.idle": "2024-03-26T09:24:55.123686Z",
     "shell.execute_reply": "2024-03-26T09:24:55.121970Z",
     "shell.execute_reply.started": "2024-03-26T09:24:55.088419Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE GRANDE: 1.7583798882681565\n",
      "R2 Score GRANDE: 0.8311929764673647\n",
      "\n",
      "\n",
      "MAE XGB: 2.3254189944134076\n",
      "R2 Score XGB: 0.7261986930299364\n",
      "\n",
      "\n",
      "MAE CatBoost: 4.0139664804469275\n",
      "R2 Score CatBoost: 0.3879855183782981\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if args['objective'] == 'binary':\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.round(preds_grande[:,1]))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.round(preds_grande[:,1]), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_grande[:,1], average='macro', multi_class='ovo')\n",
    "\n",
    "    print('Accuracy GRANDE:', accuracy)\n",
    "    print('F1 Score GRANDE:', f1_score)\n",
    "    print('ROC AUC GRANDE:', roc_auc)\n",
    "    print('\\n')\n",
    "\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.round(preds_xgb[:,1]))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.round(preds_xgb[:,1]), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_xgb[:,1], average='macro', multi_class='ovo')\n",
    "\n",
    "    print('Accuracy XGB:', accuracy)\n",
    "    print('F1 Score XGB:', f1_score)\n",
    "    print('ROC AUC XGB:', roc_auc)\n",
    "    print('\\n')\n",
    "\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.round(preds_catboost[:,1]))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.round(preds_catboost[:,1]), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_catboost[:,1], average='macro', multi_class='ovo')\n",
    "\n",
    "    print('Accuracy CatBoost:', accuracy)\n",
    "    print('F1 Score CatBoost:', f1_score)\n",
    "    print('ROC AUC CatBoost:', roc_auc)\n",
    "    print('\\n')\n",
    "elif args['objective'] == 'classification':\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.argmax(preds_grande, axis=1))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.argmax(preds_grande, axis=1), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_grande, average='macro', multi_class='ovo', labels=[i for i in range(preds_grande.shape[1])])\n",
    "\n",
    "    print('Accuracy GRANDE:', accuracy)\n",
    "    print('F1 Score GRANDE:', f1_score)\n",
    "    print('ROC AUC GRANDE:', roc_auc)\n",
    "    print('\\n')\n",
    "\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.argmax(preds_xgb, axis=1))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.argmax(preds_xgb, axis=1), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_xgb, average='macro', multi_class='ovo', labels=[i for i in range(preds_grande.shape[1])])\n",
    "\n",
    "    print('Accuracy XGB:', accuracy)\n",
    "    print('F1 Score XGB:', f1_score)\n",
    "    print('ROC AUC XGB:', roc_auc)\n",
    "    print('\\n')\n",
    "\n",
    "    accuracy = sklearn.metrics.accuracy_score(y_test, np.argmax(preds_catboost, axis=1))\n",
    "    f1_score = sklearn.metrics.f1_score(y_test, np.argmax(preds_catboost, axis=1), average='macro')\n",
    "    roc_auc = sklearn.metrics.roc_auc_score(y_test, preds_catboost, average='macro', multi_class='ovo', labels=[i for i in range(preds_grande.shape[1])])\n",
    "\n",
    "    print('Accuracy CatBoost:', accuracy)\n",
    "    print('F1 Score CatBoost:', f1_score)\n",
    "    print('ROC AUC CatBoost:', roc_auc)\n",
    "    print('\\n')\n",
    "else:\n",
    "    mean_absolute_error = sklearn.metrics.mean_absolute_error(y_test, np.round(preds_grande))\n",
    "    r2_score = sklearn.metrics.r2_score(y_test, np.round(preds_grande))\n",
    "\n",
    "    print('MAE GRANDE:', mean_absolute_error)\n",
    "    print('R2 Score GRANDE:', r2_score)\n",
    "    print('\\n')\n",
    "\n",
    "    mean_absolute_error = sklearn.metrics.mean_absolute_error(y_test, np.round(preds_xgb))\n",
    "    r2_score = sklearn.metrics.r2_score(y_test, np.round(preds_xgb))\n",
    "\n",
    "    print('MAE XGB:', mean_absolute_error)\n",
    "    print('R2 Score XGB:', r2_score)\n",
    "    print('\\n')\n",
    "\n",
    "    mean_absolute_error = sklearn.metrics.mean_absolute_error(y_test, np.round(preds_catboost))\n",
    "    r2_score = sklearn.metrics.r2_score(y_test, np.round(preds_catboost))\n",
    "\n",
    "    print('MAE CatBoost:', mean_absolute_error)\n",
    "    print('R2 Score CatBoost:', r2_score)\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ef0e2bc0-9b43-4e5c-9f55-d1dae0de8930",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-26T09:24:55.126287Z",
     "iopub.status.busy": "2024-03-26T09:24:55.125535Z",
     "iopub.status.idle": "2024-03-26T09:24:57.649935Z",
     "shell.execute_reply": "2024-03-26T09:24:57.647990Z",
     "shell.execute_reply.started": "2024-03-26T09:24:55.126233Z"
    }
   },
   "outputs": [],
   "source": [
    "from numba import cuda \n",
    "device = cuda.get_current_device()\n",
    "device.reset()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}