perplexity-baseline-phi2-1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "52dc1e8d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:17:31.946049Z",
     "iopub.status.busy": "2024-03-20T17:17:31.945694Z",
     "iopub.status.idle": "2024-03-20T17:17:31.949976Z",
     "shell.execute_reply": "2024-03-20T17:17:31.949268Z"
    },
    "papermill": {
     "duration": 0.016798,
     "end_time": "2024-03-20T17:17:31.951781",
     "exception": false,
     "start_time": "2024-03-20T17:17:31.934983",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#CREDITS\n",
    "#https://www.kaggle.com/code/philippsinger/h2ogpt-perplexity-ranking\n",
    "#https://www.kaggle.com/code/aatiffraz/prompt-prediction-w-mixtral-mistral7b-gemma-llama/notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a133c3a9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:17:31.971079Z",
     "iopub.status.busy": "2024-03-20T17:17:31.970737Z",
     "iopub.status.idle": "2024-03-20T17:18:26.724949Z",
     "shell.execute_reply": "2024-03-20T17:18:26.723726Z"
    },
    "papermill": {
     "duration": 54.766836,
     "end_time": "2024-03-20T17:18:26.727505",
     "exception": false,
     "start_time": "2024-03-20T17:17:31.960669",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "# If you want to save 4-bit models, make sure to have `bitsandbytes>=0.41.3` installed\n",
    "!pip install --no-index /kaggle/input/making-wheels-of-necessary-packages-for-hf-llms/bitsandbytes-0.42.0-py3-none-any.whl --find-links=/kaggle/input/making-wheels-of-necessary-packages-for-hf-llms\n",
    "!pip install --no-index /kaggle/input/making-wheels-of-necessary-packages-for-hf-llms/accelerate-0.27.2-py3-none-any.whl --find-links=/kaggle/input/making-wheels-of-necessary-packages-for-hf-llms\n",
    "!pip install --no-index /kaggle/input/making-wheels-of-necessary-packages-for-hf-llms/transformers-4.38.1-py3-none-any.whl --find-links=/kaggle/input/making-wheels-of-necessary-packages-for-hf-llms\n",
    "!pip install --no-index /kaggle/input/making-wheels-of-necessary-packages-for-hf-llms/optimum-1.17.1-py3-none-any.whl --find-links=/kaggle/input/making-wheels-of-necessary-packages-for-hf-llms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "160a1648",
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2024-03-20T17:18:26.746808Z",
     "iopub.status.busy": "2024-03-20T17:18:26.746489Z",
     "iopub.status.idle": "2024-03-20T17:18:33.554260Z",
     "shell.execute_reply": "2024-03-20T17:18:33.553525Z"
    },
    "papermill": {
     "duration": 6.819924,
     "end_time": "2024-03-20T17:18:33.556537",
     "exception": false,
     "start_time": "2024-03-20T17:18:26.736613",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import bitsandbytes\n",
    "import accelerate\n",
    "import transformers\n",
    "import optimum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0fc6976e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:18:33.576039Z",
     "iopub.status.busy": "2024-03-20T17:18:33.575537Z",
     "iopub.status.idle": "2024-03-20T17:19:49.454169Z",
     "shell.execute_reply": "2024-03-20T17:19:49.453367Z"
    },
    "papermill": {
     "duration": 75.890863,
     "end_time": "2024-03-20T17:19:49.456526",
     "exception": false,
     "start_time": "2024-03-20T17:18:33.565663",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fc550ad04fb4d8da8f19d2ae69d9670",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig\n",
    "\n",
    "\n",
    "# Comment/Uncomment and use as per wish\n",
    "\n",
    "#MODEL_PATH = \"/kaggle/input/gemma/transformers/7b-it/1\"\n",
    "#MODEL_PATH = \"/kaggle/input/gemma/transformers/2b-it/2\"\n",
    "#MODEL_PATH = \"/kaggle/input/mistral/pytorch/7b-instruct-v0.1-hf/1\"\n",
    "# MODEL_PATH = \"/kaggle/input/mixtral/pytorch/8x7b-instruct-v0.1-hf/1\"\n",
    "MODEL_PATH = \"/kaggle/input/phi/transformers/2/1\"\n",
    "\n",
    "\n",
    "# https://huggingface.co/blog/4bit-transformers-bitsandbytes\n",
    "# https://huggingface.co/docs/transformers/v4.38.1/en/quantization#compute-data-type\n",
    "quantization_config = BitsAndBytesConfig(\n",
    "    load_in_4bit = True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    "    bnb_4bit_compute_dtype=torch.bfloat16,\n",
    "    bnb_4bit_use_double_quant=True,\n",
    ")\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, model_max_length=3072)\n",
    "tokenizer.padding_side = \"left\"\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    MODEL_PATH,\n",
    "    device_map = \"auto\",\n",
    "    trust_remote_code = True,\n",
    "    quantization_config=quantization_config,\n",
    ")\n",
    "\n",
    "# model = model.to_bettertransformer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ef01e0c0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:19:49.476854Z",
     "iopub.status.busy": "2024-03-20T17:19:49.476092Z",
     "iopub.status.idle": "2024-03-20T17:20:03.231386Z",
     "shell.execute_reply": "2024-03-20T17:20:03.230391Z"
    },
    "papermill": {
     "duration": 13.767438,
     "end_time": "2024-03-20T17:20:03.233491",
     "exception": false,
     "start_time": "2024-03-20T17:19:49.466053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-20 17:19:51.997402: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-03-20 17:19:51.997533: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-03-20 17:19:52.130104: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>rewrite_prompt</th>\n",
       "      <th>rewritten_text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-1</th>\n",
       "      <td>The competition dataset comprises text passage...</td>\n",
       "      <td>Convert this into a sea shanty: \"\"\"The competi...</td>\n",
       "      <td>Here is your shanty: (Verse 1) The text is rew...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        original_text  \\\n",
       "id                                                      \n",
       "-1  The competition dataset comprises text passage...   \n",
       "\n",
       "                                       rewrite_prompt  \\\n",
       "id                                                      \n",
       "-1  Convert this into a sea shanty: \"\"\"The competi...   \n",
       "\n",
       "                                       rewritten_text  \n",
       "id                                                     \n",
       "-1  Here is your shanty: (Verse 1) The text is rew...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from string import Template\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\")\n",
    "\n",
    "import torch\n",
    "from transformers import pipeline, AutoTokenizer\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "data_path = Path('/kaggle/input/llm-prompt-recovery')\n",
    "\n",
    "if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n",
    "    test = pd.read_csv(data_path / 'test.csv', index_col='id')\n",
    "    test[\"rewrite_prompt\"] = \"-\"\n",
    "else:\n",
    "    test = pd.read_csv(data_path / 'train.csv', index_col='id')\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "579b6793",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:20:03.258151Z",
     "iopub.status.busy": "2024-03-20T17:20:03.257814Z",
     "iopub.status.idle": "2024-03-20T17:20:03.265646Z",
     "shell.execute_reply": "2024-03-20T17:20:03.264756Z"
    },
    "papermill": {
     "duration": 0.021764,
     "end_time": "2024-03-20T17:20:03.267520",
     "exception": false,
     "start_time": "2024-03-20T17:20:03.245756",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "class Perplexity(nn.Module):\n",
    "    def __init__(self, reduce: bool = True):\n",
    "        super().__init__()\n",
    "        self.loss_fn = nn.CrossEntropyLoss()\n",
    "        self.reduce = reduce\n",
    "\n",
    "    def forward(self, logits, labels):\n",
    "        shift_logits = logits[..., :-1, :].contiguous()\n",
    "        shift_labels = labels[..., 1:].contiguous()\n",
    "\n",
    "        perplexity = []\n",
    "        for i in range(labels.shape[0]):\n",
    "            perplexity.append(self.loss_fn(shift_logits[i], shift_labels[i]))\n",
    "        perplexity = torch.stack(perplexity, dim=0)\n",
    "        #perplexity = torch.exp(perplexity)\n",
    "        if self.reduce:\n",
    "            perplexity = torch.mean(perplexity)\n",
    "        return perplexity \n",
    "    \n",
    "perp = Perplexity()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0c9a8929",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:20:03.288932Z",
     "iopub.status.busy": "2024-03-20T17:20:03.288392Z",
     "iopub.status.idle": "2024-03-20T17:20:03.293169Z",
     "shell.execute_reply": "2024-03-20T17:20:03.292301Z"
    },
    "papermill": {
     "duration": 0.018208,
     "end_time": "2024-03-20T17:20:03.295194",
     "exception": false,
     "start_time": "2024-03-20T17:20:03.276986",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def format_prompt(row, prompt):\n",
    "    prompt_ = f\"\"\"<start_of_turn>user\n",
    "{prompt}\n",
    "{row[\"original_text\"]}<end_of_turn>\n",
    "<start_of_turn>model\n",
    "{row[\"rewritten_text\"]}<end_of_turn>\"\"\"\n",
    "    return prompt_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e55d1674",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:20:03.314979Z",
     "iopub.status.busy": "2024-03-20T17:20:03.314714Z",
     "iopub.status.idle": "2024-03-20T17:20:03.318851Z",
     "shell.execute_reply": "2024-03-20T17:20:03.318005Z"
    },
    "papermill": {
     "duration": 0.016228,
     "end_time": "2024-03-20T17:20:03.320713",
     "exception": false,
     "start_time": "2024-03-20T17:20:03.304485",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "rewrite_prompts = [\n",
    "    'Please improve the following text using the writing style of, maintaining the original meaning but altering the tone, diction, and stylistic elements to match the new style.Enhance the clarity, elegance, and impact of the following text by adopting the writing style of , ensuring the core message remains intact while transforming the tone, word choice, and stylistic features to align with the specified style.',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f9be5021",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:20:03.340310Z",
     "iopub.status.busy": "2024-03-20T17:20:03.340035Z",
     "iopub.status.idle": "2024-03-20T17:20:04.441781Z",
     "shell.execute_reply": "2024-03-20T17:20:04.440910Z"
    },
    "papermill": {
     "duration": 1.115614,
     "end_time": "2024-03-20T17:20:04.445667",
     "exception": false,
     "start_time": "2024-03-20T17:20:03.330053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "213071cc61a44f5680877808a883b468",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Please improve the following text using the writing style of, maintaining the original meaning but altering the tone, diction, and stylistic elements to match the new style.Enhance the clarity, elegance, and impact of the following text by adopting the writing style of , ensuring the core message remains intact while transforming the tone, word choice, and stylistic features to align with the specified style.']\n"
     ]
    }
   ],
   "source": [
    "preds = []\n",
    "\n",
    "for idx, row in tqdm(test.iterrows(), total=len(test)):\n",
    "        \n",
    "    \n",
    "    with torch.no_grad():\n",
    "        perps = []\n",
    "        samples = []\n",
    "        for prompt in rewrite_prompts:\n",
    "            samples.append(format_prompt(row, prompt))\n",
    "        inputs = tokenizer(samples, return_tensors=\"pt\", add_special_tokens=False, padding=True, truncation=True).to(\"cuda\")\n",
    "\n",
    "        output = model(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"])\n",
    "        output = output.logits\n",
    "        labels = inputs[\"input_ids\"]\n",
    "        labels.masked_fill_(~inputs[\"attention_mask\"].bool(), -100)\n",
    "        for j in range(len(rewrite_prompts)):\n",
    "            p = perp(output[j].unsqueeze(0), labels[j].unsqueeze(0))\n",
    "            perps.append(p.detach().cpu())\n",
    "            \n",
    "        del inputs\n",
    "        del labels\n",
    "        del output\n",
    "        del p\n",
    "\n",
    "    perps = np.array(perps)\n",
    "        \n",
    "    predictions = [np.array(rewrite_prompts)[np.argsort(perps)][0]]\n",
    "    preds.append(predictions[0])\n",
    "    print(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a7cd8a1b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:20:04.467475Z",
     "iopub.status.busy": "2024-03-20T17:20:04.466765Z",
     "iopub.status.idle": "2024-03-20T17:20:04.475673Z",
     "shell.execute_reply": "2024-03-20T17:20:04.474889Z"
    },
    "papermill": {
     "duration": 0.021868,
     "end_time": "2024-03-20T17:20:04.477547",
     "exception": false,
     "start_time": "2024-03-20T17:20:04.455679",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission = pd.read_csv(data_path / 'sample_submission.csv')\n",
    "submission[\"rewrite_prompt\"] = preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "88dc32c9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:20:04.498160Z",
     "iopub.status.busy": "2024-03-20T17:20:04.497856Z",
     "iopub.status.idle": "2024-03-20T17:20:04.506178Z",
     "shell.execute_reply": "2024-03-20T17:20:04.505335Z"
    },
    "papermill": {
     "duration": 0.020875,
     "end_time": "2024-03-20T17:20:04.508081",
     "exception": false,
     "start_time": "2024-03-20T17:20:04.487206",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>rewrite_prompt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9559194</td>\n",
       "      <td>Please improve the following text using the wr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                     rewrite_prompt\n",
       "0  9559194  Please improve the following text using the wr..."
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b84fc5db",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-20T17:20:04.528882Z",
     "iopub.status.busy": "2024-03-20T17:20:04.528560Z",
     "iopub.status.idle": "2024-03-20T17:20:04.536251Z",
     "shell.execute_reply": "2024-03-20T17:20:04.535392Z"
    },
    "papermill": {
     "duration": 0.020354,
     "end_time": "2024-03-20T17:20:04.538264",
     "exception": false,
     "start_time": "2024-03-20T17:20:04.517910",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8698bfc4",
   "metadata": {
    "papermill": {
     "duration": 0.009768,
     "end_time": "2024-03-20T17:20:04.557944",
     "exception": false,
     "start_time": "2024-03-20T17:20:04.548176",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 7806901,
     "sourceId": 67121,
     "sourceType": "competition"
    },
    {
     "datasetId": 4506214,
     "sourceId": 7747717,
     "sourceType": "datasetVersion"
    },
    {
     "sourceId": 164836055,
     "sourceType": "kernelVersion"
    },
    {
     "modelInstanceId": 3093,
     "sourceId": 4298,
     "sourceType": "modelInstanceVersion"
    },
    {
     "modelInstanceId": 3097,
     "sourceId": 4302,
     "sourceType": "modelInstanceVersion"
    },
    {
     "modelInstanceId": 3900,
     "sourceId": 5112,
     "sourceType": "modelInstanceVersion"
    },
    {
     "modelInstanceId": 4761,
     "sourceId": 5994,
     "sourceType": "modelInstanceVersion"
    },
    {
     "modelInstanceId": 8658,
     "sourceId": 10716,
     "sourceType": "modelInstanceVersion"
    },
    {
     "modelInstanceId": 8332,
     "sourceId": 11261,
     "sourceType": "modelInstanceVersion"
    },
    {
     "modelInstanceId": 8318,
     "sourceId": 11382,
     "sourceType": "modelInstanceVersion"
    }
   ],
   "dockerImageVersionId": 30665,
   "isGpuEnabled": true,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 158.273735,
   "end_time": "2024-03-20T17:20:07.417959",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-03-20T17:17:29.144224",
   "version": "2.5.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "08a9ad995d62424f841d4b13f7cb363c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bced86e9a4174b5fbc24d9d4e3aa0a1b",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_0cdb59c023594104a413b03392d76403",
       "value": 1.0
      }
     },
     "0cdb59c023594104a413b03392d76403": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "0fc550ad04fb4d8da8f19d2ae69d9670": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_5992077c5dd34e9aa100a18feb133bcb",
        "IPY_MODEL_ab45d1d34cb944c4953f42fac9c2200b",
        "IPY_MODEL_a4946fd0aebb4098ba4ed58897a481e8"
       ],
       "layout": "IPY_MODEL_d0effdff34a14533837cf48ee8d78ef4"
      }
     },
     "104981cf613745a7adccab4b55f84bba": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "1d09a19ac235446da1a4a9e106c7e65a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "213071cc61a44f5680877808a883b468": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_aec6f942c69f4ac19df74fd81db1d4b6",
        "IPY_MODEL_08a9ad995d62424f841d4b13f7cb363c",
        "IPY_MODEL_4b9d454457ba45efb2f87933f3b80c78"
       ],
       "layout": "IPY_MODEL_9f3ce70d1bc3492884cb8c0e0eda6678"
      }
     },
     "288945b07c7346b5b671b3a8e753a0e1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "386cced475b344399cad107b0e82e040": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4b9d454457ba45efb2f87933f3b80c78": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_386cced475b344399cad107b0e82e040",
       "placeholder": "​",
       "style": "IPY_MODEL_e47641b44ce547dbb9786fcd933f9114",
       "value": " 1/1 [00:01&lt;00:00,  1.09s/it]"
      }
     },
     "5992077c5dd34e9aa100a18feb133bcb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f1d0991f4dfc454f85d33a6532bd3d2a",
       "placeholder": "​",
       "style": "IPY_MODEL_7ad2d7e3680b406a98ed64fd266841d5",
       "value": "Loading checkpoint shards: 100%"
      }
     },
     "7ad2d7e3680b406a98ed64fd266841d5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "9f3ce70d1bc3492884cb8c0e0eda6678": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a4946fd0aebb4098ba4ed58897a481e8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fdfd1172d2224f36a4f45ae00eab7928",
       "placeholder": "​",
       "style": "IPY_MODEL_104981cf613745a7adccab4b55f84bba",
       "value": " 2/2 [01:13&lt;00:00, 31.69s/it]"
      }
     },
     "ab45d1d34cb944c4953f42fac9c2200b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_288945b07c7346b5b671b3a8e753a0e1",
       "max": 2.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_b1fbef1039a4426f8877a6d2a1b6ede0",
       "value": 2.0
      }
     },
     "aec6f942c69f4ac19df74fd81db1d4b6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1d09a19ac235446da1a4a9e106c7e65a",
       "placeholder": "​",
       "style": "IPY_MODEL_e446672c3ea7451ebb3ea65441b44e5e",
       "value": "100%"
      }
     },
     "b1fbef1039a4426f8877a6d2a1b6ede0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "bced86e9a4174b5fbc24d9d4e3aa0a1b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d0effdff34a14533837cf48ee8d78ef4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e446672c3ea7451ebb3ea65441b44e5e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e47641b44ce547dbb9786fcd933f9114": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f1d0991f4dfc454f85d33a6532bd3d2a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fdfd1172d2224f36a4f45ae00eab7928": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}