0-61-llmpr-phi2-sft-model-training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "66a4b2b9",
   "metadata": {
    "papermill": {
     "duration": 0.006214,
     "end_time": "2024-04-01T19:47:43.487314",
     "exception": false,
     "start_time": "2024-04-01T19:47:43.481100",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Phi2 SFT training baseline\n",
    "\n",
    "## data\n",
    "\n",
    "use all public data, but I dropped the dupilcate rewrite prompts.\n",
    "\n",
    "## hyperparamters\n",
    "\n",
    "epoch: 5\n",
    "\n",
    "batch size: 2\n",
    "\n",
    "gradient_accumulation_steps: 8\n",
    "\n",
    "max_seq_length: 1024\n",
    "\n",
    "learing rate: 1e-4\n",
    "\n",
    "\n",
    "inference notebook click [here](https://www.kaggle.com/code/mozhiwenmzw/0-61-llmpr-phi2-sft-model-generate-infer?scriptVersionId=169380324)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c7d8e020",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-04-01T19:47:43.500472Z",
     "iopub.status.busy": "2024-04-01T19:47:43.499780Z",
     "iopub.status.idle": "2024-04-01T19:48:25.834955Z",
     "shell.execute_reply": "2024-04-01T19:48:25.833865Z"
    },
    "papermill": {
     "duration": 42.344591,
     "end_time": "2024-04-01T19:48:25.837728",
     "exception": false,
     "start_time": "2024-04-01T19:47:43.493137",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install -Uq /kaggle/input/llm-whls/bitsandbytes-0.41.1-py3-none-any.whl\n",
    "!pip install -Uq /kaggle/input/llm-whls/peft-0.4.0-py3-none-any.whl\n",
    "!pip install -Uq /kaggle/input/library-off-for-llm/transformers-4.38.2-py3-none-any.whl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4b3a3e2e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:48:25.851272Z",
     "iopub.status.busy": "2024-04-01T19:48:25.850937Z",
     "iopub.status.idle": "2024-04-01T19:48:38.124898Z",
     "shell.execute_reply": "2024-04-01T19:48:38.123693Z"
    },
    "papermill": {
     "duration": 12.283549,
     "end_time": "2024-04-01T19:48:38.127430",
     "exception": false,
     "start_time": "2024-04-01T19:48:25.843881",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install -Uq /kaggle/input/llm-whls/trl-0.5.0-py3-none-any.whl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dcf9c8b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:48:38.141628Z",
     "iopub.status.busy": "2024-04-01T19:48:38.141326Z",
     "iopub.status.idle": "2024-04-01T19:48:56.021228Z",
     "shell.execute_reply": "2024-04-01T19:48:56.020458Z"
    },
    "papermill": {
     "duration": 17.889781,
     "end_time": "2024-04-01T19:48:56.023449",
     "exception": false,
     "start_time": "2024-04-01T19:48:38.133668",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-04-01 19:48:48.774314: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-04-01 19:48:48.774409: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-04-01 19:48:48.897909: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from datasets import Dataset\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
    "from transformers import TrainingArguments\n",
    "\n",
    "from trl import SFTTrainer, DataCollatorForCompletionOnlyLM\n",
    "from peft import LoraConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b1b0c135",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:48:56.037492Z",
     "iopub.status.busy": "2024-04-01T19:48:56.036468Z",
     "iopub.status.idle": "2024-04-01T19:48:56.040984Z",
     "shell.execute_reply": "2024-04-01T19:48:56.040306Z"
    },
    "papermill": {
     "duration": 0.013228,
     "end_time": "2024-04-01T19:48:56.042824",
     "exception": false,
     "start_time": "2024-04-01T19:48:56.029596",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "exp_name = 'phi2_public_data_sft'\n",
    "data_path = '/kaggle/input/llmpr-public-10k-unique/public_10k_unique_rewrite_prompt.csv'\n",
    "model_path = '/kaggle/input/phi/transformers/2/1'\n",
    "output_path = f'outputs'\n",
    "model_save_path =  f'{exp_name}_adapter'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "60549a25",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:48:56.055431Z",
     "iopub.status.busy": "2024-04-01T19:48:56.055166Z",
     "iopub.status.idle": "2024-04-01T19:48:56.059010Z",
     "shell.execute_reply": "2024-04-01T19:48:56.058212Z"
    },
    "papermill": {
     "duration": 0.012302,
     "end_time": "2024-04-01T19:48:56.060945",
     "exception": false,
     "start_time": "2024-04-01T19:48:56.048643",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "epochs=5\n",
    "batch_size=1 # 2 \n",
    "max_seq_length=512 # 1024 \n",
    "lr = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "102f68d1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:48:56.073684Z",
     "iopub.status.busy": "2024-04-01T19:48:56.073430Z",
     "iopub.status.idle": "2024-04-01T19:48:56.758366Z",
     "shell.execute_reply": "2024-04-01T19:48:56.757500Z"
    },
    "papermill": {
     "duration": 0.693865,
     "end_time": "2024-04-01T19:48:56.760700",
     "exception": false,
     "start_time": "2024-04-01T19:48:56.066835",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(data_path)\n",
    "train_df, val_df = train_test_split(df, test_size=0.3, random_state=42)\n",
    "train_df = train_df.reset_index(drop=True)\n",
    "val_df = val_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8f7ee657",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:48:56.775047Z",
     "iopub.status.busy": "2024-04-01T19:48:56.774760Z",
     "iopub.status.idle": "2024-04-01T19:48:56.855008Z",
     "shell.execute_reply": "2024-04-01T19:48:56.854244Z"
    },
    "papermill": {
     "duration": 0.090513,
     "end_time": "2024-04-01T19:48:56.857308",
     "exception": false,
     "start_time": "2024-04-01T19:48:56.766795",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_ds = Dataset.from_pandas(train_df)\n",
    "val_ds = Dataset.from_pandas(val_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d8378019",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:48:56.870624Z",
     "iopub.status.busy": "2024-04-01T19:48:56.870337Z",
     "iopub.status.idle": "2024-04-01T19:48:57.049407Z",
     "shell.execute_reply": "2024-04-01T19:48:57.048525Z"
    },
    "papermill": {
     "duration": 0.18791,
     "end_time": "2024-04-01T19:48:57.051364",
     "exception": false,
     "start_time": "2024-04-01T19:48:56.863454",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    model_path,\n",
    "    )\n",
    "tokenizer.pad_token = tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b91cd8a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:48:57.064721Z",
     "iopub.status.busy": "2024-04-01T19:48:57.064444Z",
     "iopub.status.idle": "2024-04-01T19:48:57.069894Z",
     "shell.execute_reply": "2024-04-01T19:48:57.069255Z"
    },
    "papermill": {
     "duration": 0.014206,
     "end_time": "2024-04-01T19:48:57.071739",
     "exception": false,
     "start_time": "2024-04-01T19:48:57.057533",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "bnb_config = BitsAndBytesConfig(\n",
    "        load_in_4bit=True,\n",
    "        bnb_4bit_quant_type='nf4',\n",
    "        bnb_4bit_compute_dtype='float16',\n",
    "        bnb_4bit_use_double_quant=False,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2eb64b29",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:48:57.084486Z",
     "iopub.status.busy": "2024-04-01T19:48:57.084243Z",
     "iopub.status.idle": "2024-04-01T19:50:03.579414Z",
     "shell.execute_reply": "2024-04-01T19:50:03.578411Z"
    },
    "papermill": {
     "duration": 66.503944,
     "end_time": "2024-04-01T19:50:03.581517",
     "exception": false,
     "start_time": "2024-04-01T19:48:57.077573",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:466: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.\n",
      "  warnings.warn(\n",
      "`low_cpu_mem_usage` was None, now set to True since model is quantized.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe785780e5134d68bac28f82385c9c9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are calling `save_pretrained` to a 4-bit converted model, but your `bitsandbytes` version doesn't support it. If you want to save 4-bit models, make sure to have `bitsandbytes>=0.41.3` installed.\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForCausalLM.from_pretrained(model_path,\n",
    "                                             quantization_config=bnb_config,\n",
    "                                             trust_remote_code=True,\n",
    "                                             use_auth_token=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5f00b698",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:50:03.595638Z",
     "iopub.status.busy": "2024-04-01T19:50:03.595350Z",
     "iopub.status.idle": "2024-04-01T19:50:03.599672Z",
     "shell.execute_reply": "2024-04-01T19:50:03.598754Z"
    },
    "papermill": {
     "duration": 0.013614,
     "end_time": "2024-04-01T19:50:03.601762",
     "exception": false,
     "start_time": "2024-04-01T19:50:03.588148",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.config.gradient_checkpointing = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3def3247",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:50:03.615840Z",
     "iopub.status.busy": "2024-04-01T19:50:03.615259Z",
     "iopub.status.idle": "2024-04-01T19:50:03.620254Z",
     "shell.execute_reply": "2024-04-01T19:50:03.619436Z"
    },
    "papermill": {
     "duration": 0.014233,
     "end_time": "2024-04-01T19:50:03.622191",
     "exception": false,
     "start_time": "2024-04-01T19:50:03.607958",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def token_len(text):\n",
    "    tokenized = tokenizer(text, return_length=True)\n",
    "    length = tokenized['length'][0]\n",
    "    return length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "07b07ead",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:50:03.635805Z",
     "iopub.status.busy": "2024-04-01T19:50:03.635491Z",
     "iopub.status.idle": "2024-04-01T19:50:03.642243Z",
     "shell.execute_reply": "2024-04-01T19:50:03.641421Z"
    },
    "papermill": {
     "duration": 0.015899,
     "end_time": "2024-04-01T19:50:03.644248",
     "exception": false,
     "start_time": "2024-04-01T19:50:03.628349",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def formatting_prompts_func(example):\n",
    "    output_texts = []\n",
    "    for i in range(len(example['rewritten_text'])):\n",
    "        ori_text = example['original_text'][i]\n",
    "        rew_text = example['rewritten_text'][i]\n",
    "        rew_prompt = example['rewrite_prompt'][i]\n",
    "        text = f\"Instruct: Original Text:{ori_text}\\nRewritten Text:{rew_text}\\nWrite a prompt that was likely given to the LLM to rewrite original text into rewritten text.Output: {rew_prompt}\"\n",
    "        if token_len(text) > max_seq_length:\n",
    "            continue\n",
    "        output_texts.append(text)\n",
    "    return output_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3dd154d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:50:03.658166Z",
     "iopub.status.busy": "2024-04-01T19:50:03.657534Z",
     "iopub.status.idle": "2024-04-01T19:50:03.665570Z",
     "shell.execute_reply": "2024-04-01T19:50:03.664243Z"
    },
    "papermill": {
     "duration": 0.01869,
     "end_time": "2024-04-01T19:50:03.669126",
     "exception": false,
     "start_time": "2024-04-01T19:50:03.650436",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "response_template = \"Output:\"\n",
    "collator = DataCollatorForCompletionOnlyLM(response_template=response_template, \n",
    "                                           tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ccb460cf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:50:03.683251Z",
     "iopub.status.busy": "2024-04-01T19:50:03.682609Z",
     "iopub.status.idle": "2024-04-01T19:50:03.687642Z",
     "shell.execute_reply": "2024-04-01T19:50:03.686816Z"
    },
    "papermill": {
     "duration": 0.014422,
     "end_time": "2024-04-01T19:50:03.689791",
     "exception": false,
     "start_time": "2024-04-01T19:50:03.675369",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "peft_config = LoraConfig(\n",
    "    r=16,\n",
    "    lora_alpha=32,\n",
    "    lora_dropout=0.05,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    "    target_modules= [\"q_proj\", \"k_proj\", \"v_proj\", \"dense\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f1d06270",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:50:03.703462Z",
     "iopub.status.busy": "2024-04-01T19:50:03.703125Z",
     "iopub.status.idle": "2024-04-01T19:50:03.711113Z",
     "shell.execute_reply": "2024-04-01T19:50:03.710346Z"
    },
    "papermill": {
     "duration": 0.016922,
     "end_time": "2024-04-01T19:50:03.713013",
     "exception": false,
     "start_time": "2024-04-01T19:50:03.696091",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir = output_path,\n",
    "    fp16=True,\n",
    "    learning_rate=lr,\n",
    "    optim=\"adafactor\",\n",
    "    num_train_epochs=epochs,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size*2,\n",
    "    gradient_accumulation_steps=8,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    save_total_limit=1,\n",
    "    logging_steps=50,\n",
    "    lr_scheduler_type=\"cosine\",\n",
    "    warmup_ratio=0.1,\n",
    "    weight_decay=0.01,\n",
    "    report_to='none',\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=\"eval_loss\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "159eb171",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:50:03.726471Z",
     "iopub.status.busy": "2024-04-01T19:50:03.726184Z",
     "iopub.status.idle": "2024-04-01T19:50:41.360429Z",
     "shell.execute_reply": "2024-04-01T19:50:41.359313Z"
    },
    "papermill": {
     "duration": 37.643457,
     "end_time": "2024-04-01T19:50:41.362588",
     "exception": false,
     "start_time": "2024-04-01T19:50:03.719131",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/peft/utils/other.py:102: FutureWarning: prepare_model_for_int8_training is deprecated and will be removed in a future version. Use prepare_model_for_kbit_training instead.\n",
      "  warnings.warn(\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36f91f33376d428280c3132bbc521d98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (2159 > 2048). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "84b53eb01f954e838cf7721223a5e4f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
      "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    args = args,\n",
    "    max_seq_length=max_seq_length,\n",
    "    train_dataset=train_ds,\n",
    "    eval_dataset=val_ds,\n",
    "    formatting_func=formatting_prompts_func,\n",
    "    data_collator=collator,\n",
    "    peft_config=peft_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8834beb2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-01T19:50:41.378040Z",
     "iopub.status.busy": "2024-04-01T19:50:41.377320Z",
     "iopub.status.idle": "2024-04-02T04:39:41.943084Z",
     "shell.execute_reply": "2024-04-02T04:39:41.942195Z"
    },
    "papermill": {
     "duration": 31740.57536,
     "end_time": "2024-04-02T04:39:41.944977",
     "exception": false,
     "start_time": "2024-04-01T19:50:41.369617",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2640' max='2640' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2640/2640 8:48:49, Epoch 4/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.172100</td>\n",
       "      <td>1.096457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.016100</td>\n",
       "      <td>0.979691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.861600</td>\n",
       "      <td>0.930683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.778500</td>\n",
       "      <td>0.920767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.822900</td>\n",
       "      <td>0.917286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=2640, training_loss=1.0022892092213487, metrics={'train_runtime': 31740.2301, 'train_samples_per_second': 0.666, 'train_steps_per_second': 0.083, 'total_flos': 1.08150631450368e+17, 'train_loss': 1.0022892092213487, 'epoch': 5.0})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0b6e8924",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-02T04:39:41.961468Z",
     "iopub.status.busy": "2024-04-02T04:39:41.960895Z",
     "iopub.status.idle": "2024-04-02T04:39:42.178464Z",
     "shell.execute_reply": "2024-04-02T04:39:42.177498Z"
    },
    "papermill": {
     "duration": 0.227946,
     "end_time": "2024-04-02T04:39:42.180567",
     "exception": false,
     "start_time": "2024-04-02T04:39:41.952621",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('phi2_public_data_sft_adapter/tokenizer_config.json',\n",
       " 'phi2_public_data_sft_adapter/special_tokens_map.json',\n",
       " 'phi2_public_data_sft_adapter/vocab.json',\n",
       " 'phi2_public_data_sft_adapter/merges.txt',\n",
       " 'phi2_public_data_sft_adapter/added_tokens.json',\n",
       " 'phi2_public_data_sft_adapter/tokenizer.json')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.save_model(model_save_path)\n",
    "tokenizer.save_pretrained(model_save_path)"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 7806901,
     "sourceId": 67121,
     "sourceType": "competition"
    },
    {
     "datasetId": 3600418,
     "sourceId": 6572938,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4661387,
     "sourceId": 7930545,
     "sourceType": "datasetVersion"
    },
    {
     "sourceId": 164964691,
     "sourceType": "kernelVersion"
    },
    {
     "modelInstanceId": 8658,
     "sourceId": 10716,
     "sourceType": "modelInstanceVersion"
    }
   ],
   "dockerImageVersionId": 30674,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 31924.676729,
   "end_time": "2024-04-02T04:39:45.528723",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-04-01T19:47:40.851994",
   "version": "2.5.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "05637d59fa8344db9e14166aa3213dd8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "18061c69cfbe4cbf9218f5b07c01ab99": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1adba7d1189d4b2d96a432524587912a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "300a43e36e894ee1bfb776c12031095f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "307a9f84b5a44649ab0e8e8a7537bea9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "36954fe23113499d8c769b5b935ce573": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1adba7d1189d4b2d96a432524587912a",
       "placeholder": "​",
       "style": "IPY_MODEL_8c6121154c1b4bd2855d660542b04162",
       "value": " 2/2 [01:05&lt;00:00, 28.34s/it]"
      }
     },
     "36f91f33376d428280c3132bbc521d98": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_579a8e2a7c954f14871c1c5ea930a2d7",
        "IPY_MODEL_c08a860c638348c59aa724455d213709",
        "IPY_MODEL_f9e47587d0ac446c92a414795207b49f"
       ],
       "layout": "IPY_MODEL_a9987ac7f8274330b10749b18de88854"
      }
     },
     "3cc2cb23b5994168bb0157624dc18f89": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e82b5138fd734a1ea1731d9d2c4f3cb0",
       "max": 4.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_f1a356f6e1854fea8e05a4fa18106400",
       "value": 4.0
      }
     },
     "48eca7f944a4427aa22c36a807f31a3f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "579a8e2a7c954f14871c1c5ea930a2d7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_48eca7f944a4427aa22c36a807f31a3f",
       "placeholder": "​",
       "style": "IPY_MODEL_8eba9af154334e758a77700a939d5505",
       "value": "100%"
      }
     },
     "5a70cf709d7d4c17aecd3b1234b12279": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5acab3e4cdc944b595d343b1b75ebc2e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_5a70cf709d7d4c17aecd3b1234b12279",
       "max": 2.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_307a9f84b5a44649ab0e8e8a7537bea9",
       "value": 2.0
      }
     },
     "60bf85e1f79048c7aee45ba017b40c84": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_05637d59fa8344db9e14166aa3213dd8",
       "placeholder": "​",
       "style": "IPY_MODEL_ba021b1b25004e0881a9517ef6b0b5f9",
       "value": "Loading checkpoint shards: 100%"
      }
     },
     "63e9dd14e0d44bfebfd4aa3bed702eaa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "84b53eb01f954e838cf7721223a5e4f5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_c3879860b3fb407db11502192a6c3d35",
        "IPY_MODEL_3cc2cb23b5994168bb0157624dc18f89",
        "IPY_MODEL_c7abef62ed44434f9955a1cbb35d48ce"
       ],
       "layout": "IPY_MODEL_90c62770de654b51bb1e246c42909cb5"
      }
     },
     "8c6121154c1b4bd2855d660542b04162": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "8eba9af154334e758a77700a939d5505": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "90c62770de654b51bb1e246c42909cb5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a942a103e5324509b3c56afca109495f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "a9987ac7f8274330b10749b18de88854": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b28be3ba43f6447abd2af56a82892d61": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b7480d7ad83246d59e253e96ad1a6d79": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ba021b1b25004e0881a9517ef6b0b5f9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "c08a860c638348c59aa724455d213709": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_da745a51361b4908b33ab7b716e0864a",
       "max": 8.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_da3f1dc2b667469c97693be96b25ddff",
       "value": 8.0
      }
     },
     "c3879860b3fb407db11502192a6c3d35": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_18061c69cfbe4cbf9218f5b07c01ab99",
       "placeholder": "​",
       "style": "IPY_MODEL_300a43e36e894ee1bfb776c12031095f",
       "value": "100%"
      }
     },
     "c7abef62ed44434f9955a1cbb35d48ce": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b28be3ba43f6447abd2af56a82892d61",
       "placeholder": "​",
       "style": "IPY_MODEL_a942a103e5324509b3c56afca109495f",
       "value": " 4/4 [00:09&lt;00:00,  1.89s/ba]"
      }
     },
     "da3f1dc2b667469c97693be96b25ddff": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "da745a51361b4908b33ab7b716e0864a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e82b5138fd734a1ea1731d9d2c4f3cb0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ebde10cb0f194e62a27c12a73aa0480c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f1a356f6e1854fea8e05a4fa18106400": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "f9e47587d0ac446c92a414795207b49f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b7480d7ad83246d59e253e96ad1a6d79",
       "placeholder": "​",
       "style": "IPY_MODEL_ebde10cb0f194e62a27c12a73aa0480c",
       "value": " 8/8 [00:19&lt;00:00,  1.94s/ba]"
      }
     },
     "fe785780e5134d68bac28f82385c9c9c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_60bf85e1f79048c7aee45ba017b40c84",
        "IPY_MODEL_5acab3e4cdc944b595d343b1b75ebc2e",
        "IPY_MODEL_36954fe23113499d8c769b5b935ce573"
       ],
       "layout": "IPY_MODEL_63e9dd14e0d44bfebfd4aa3bed702eaa"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}