GPT2_Fine_Tuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sudama-inc/llm_finetuning/blob/main/GPT2_Fine_Tuning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://coherent-cast.surge.sh/Cleaned_Indian_Food_Dataset.csv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fjML75LzRhQ9",
        "outputId": "e6e39371-8fcd-49ab-bba2-bc69177b03d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-09-28 10:15:12--  https://coherent-cast.surge.sh/Cleaned_Indian_Food_Dataset.csv\n",
            "Resolving coherent-cast.surge.sh (coherent-cast.surge.sh)... 188.166.132.94\n",
            "Connecting to coherent-cast.surge.sh (coherent-cast.surge.sh)|188.166.132.94|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 11736594 (11M) [text/csv]\n",
            "Saving to: ‘Cleaned_Indian_Food_Dataset.csv’\n",
            "\n",
            "Cleaned_Indian_Food 100%[===================>]  11.19M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2023-09-28 10:15:16 (95.0 MB/s) - ‘Cleaned_Indian_Food_Dataset.csv’ saved [11736594/11736594]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification, set_seed\n",
        "from transformers import TextDataset, DataCollatorForLanguageModeling\n",
        "from transformers import Trainer, TrainingArguments, AutoModelWithLMHead"
      ],
      "metadata": {
        "id": "aQKpaCliPx5e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "RANDOM_SEED = 42\n",
        "set_seed(RANDOM_SEED)"
      ],
      "metadata": {
        "id": "IQuJM5S4QAXU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda\")"
      ],
      "metadata": {
        "id": "TeO_bcbXQk_y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "food_df = pd.read_csv(\"Cleaned_Indian_Food_Dataset.csv\")\n",
        "food_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "id": "osCZCacsShw1",
        "outputId": "79038780-bba8-4a52-bf86-5bcc37d11218"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                TranslatedRecipeName  \\\n",
              "0                               Masala Karela Recipe   \n",
              "1                         Spicy Tomato Rice (Recipe)   \n",
              "2  Ragi Semiya Upma Recipe - Ragi Millet Vermicel...   \n",
              "3  Gongura Chicken Curry Recipe - Andhra Style Go...   \n",
              "4  Andhra Style Alam Pachadi Recipe - Adrak Chutn...   \n",
              "\n",
              "                               TranslatedIngredients  TotalTimeInMins  \\\n",
              "0  1 tablespoon Red Chilli powder,3 tablespoon Gr...               45   \n",
              "1   2 teaspoon cashew - or peanuts, 1/2 Teaspoon ...               15   \n",
              "2  1 Onion - sliced,1 teaspoon White Urad Dal (Sp...               50   \n",
              "3  1/2 teaspoon Turmeric powder (Haldi),1 tablesp...               45   \n",
              "4   oil - as per use, 1 tablespoon coriander seed...               30   \n",
              "\n",
              "                Cuisine                             TranslatedInstructions  \\\n",
              "0                Indian  To begin making the Masala Karela Recipe,de-se...   \n",
              "1  South Indian Recipes  To make tomato puliogere, first cut the tomato...   \n",
              "2  South Indian Recipes  To begin making the Ragi Vermicelli Recipe, fi...   \n",
              "3                Andhra  To begin making Gongura Chicken Curry Recipe f...   \n",
              "4                Andhra  To make Andhra Style Alam Pachadi, first heat ...   \n",
              "\n",
              "                                                 URL  \\\n",
              "0  https://www.archanaskitchen.com/masala-karela-...   \n",
              "1  https://www.archanaskitchen.com/spicy-tomato-r...   \n",
              "2  https://www.archanaskitchen.com/ragi-vermicell...   \n",
              "3  https://www.archanaskitchen.com/gongura-chicke...   \n",
              "4  https://www.archanaskitchen.com/andhra-style-a...   \n",
              "\n",
              "                                 Cleaned-Ingredients  \\\n",
              "0  salt,amchur (dry mango powder),karela (bitter ...   \n",
              "1  tomato,salt,chickpea lentils,green chilli,rice...   \n",
              "2  salt,rice vermicelli noodles (thin),asafoetida...   \n",
              "3  tomato,salt,ginger,sorrel leaves (gongura),fen...   \n",
              "4  tomato,salt,ginger,red chillies,curry,asafoeti...   \n",
              "\n",
              "                                           image-url  Ingredient-count  \n",
              "0  https://www.archanaskitchen.com/images/archana...                10  \n",
              "1  https://www.archanaskitchen.com/images/archana...                12  \n",
              "2  https://www.archanaskitchen.com/images/archana...                12  \n",
              "3  https://www.archanaskitchen.com/images/archana...                15  \n",
              "4  https://www.archanaskitchen.com/images/archana...                12  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-efee6194-283c-4d7f-9ee8-95b916181ff6\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>TranslatedRecipeName</th>\n",
              "      <th>TranslatedIngredients</th>\n",
              "      <th>TotalTimeInMins</th>\n",
              "      <th>Cuisine</th>\n",
              "      <th>TranslatedInstructions</th>\n",
              "      <th>URL</th>\n",
              "      <th>Cleaned-Ingredients</th>\n",
              "      <th>image-url</th>\n",
              "      <th>Ingredient-count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Masala Karela Recipe</td>\n",
              "      <td>1 tablespoon Red Chilli powder,3 tablespoon Gr...</td>\n",
              "      <td>45</td>\n",
              "      <td>Indian</td>\n",
              "      <td>To begin making the Masala Karela Recipe,de-se...</td>\n",
              "      <td>https://www.archanaskitchen.com/masala-karela-...</td>\n",
              "      <td>salt,amchur (dry mango powder),karela (bitter ...</td>\n",
              "      <td>https://www.archanaskitchen.com/images/archana...</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Spicy Tomato Rice (Recipe)</td>\n",
              "      <td>2 teaspoon cashew - or peanuts, 1/2 Teaspoon ...</td>\n",
              "      <td>15</td>\n",
              "      <td>South Indian Recipes</td>\n",
              "      <td>To make tomato puliogere, first cut the tomato...</td>\n",
              "      <td>https://www.archanaskitchen.com/spicy-tomato-r...</td>\n",
              "      <td>tomato,salt,chickpea lentils,green chilli,rice...</td>\n",
              "      <td>https://www.archanaskitchen.com/images/archana...</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Ragi Semiya Upma Recipe - Ragi Millet Vermicel...</td>\n",
              "      <td>1 Onion - sliced,1 teaspoon White Urad Dal (Sp...</td>\n",
              "      <td>50</td>\n",
              "      <td>South Indian Recipes</td>\n",
              "      <td>To begin making the Ragi Vermicelli Recipe, fi...</td>\n",
              "      <td>https://www.archanaskitchen.com/ragi-vermicell...</td>\n",
              "      <td>salt,rice vermicelli noodles (thin),asafoetida...</td>\n",
              "      <td>https://www.archanaskitchen.com/images/archana...</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Gongura Chicken Curry Recipe - Andhra Style Go...</td>\n",
              "      <td>1/2 teaspoon Turmeric powder (Haldi),1 tablesp...</td>\n",
              "      <td>45</td>\n",
              "      <td>Andhra</td>\n",
              "      <td>To begin making Gongura Chicken Curry Recipe f...</td>\n",
              "      <td>https://www.archanaskitchen.com/gongura-chicke...</td>\n",
              "      <td>tomato,salt,ginger,sorrel leaves (gongura),fen...</td>\n",
              "      <td>https://www.archanaskitchen.com/images/archana...</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Andhra Style Alam Pachadi Recipe - Adrak Chutn...</td>\n",
              "      <td>oil - as per use, 1 tablespoon coriander seed...</td>\n",
              "      <td>30</td>\n",
              "      <td>Andhra</td>\n",
              "      <td>To make Andhra Style Alam Pachadi, first heat ...</td>\n",
              "      <td>https://www.archanaskitchen.com/andhra-style-a...</td>\n",
              "      <td>tomato,salt,ginger,red chillies,curry,asafoeti...</td>\n",
              "      <td>https://www.archanaskitchen.com/images/archana...</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-efee6194-283c-4d7f-9ee8-95b916181ff6')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-efee6194-283c-4d7f-9ee8-95b916181ff6 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-efee6194-283c-4d7f-9ee8-95b916181ff6');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-51e1d30a-02fb-4ac5-a37b-3fe54c7b1f49\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-51e1d30a-02fb-4ac5-a37b-3fe54c7b1f49')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-51e1d30a-02fb-4ac5-a37b-3fe54c7b1f49 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "food_instructions = food_df[\"TranslatedInstructions\"].tolist()\n",
        "train_data, test_data = train_test_split(food_instructions, test_size=0.2)\n",
        "print(f\"{len(train_data) = }; {len(test_data) = }\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jV1sWqg3SuTN",
        "outputId": "8a26293e-6fc1-4626-eebe-2955ad489b60"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "len(train_data) = 4750; len(test_data) = 1188\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"train_data.txt\", \"w\") as f:\n",
        "  f.writelines(train_data)\n",
        "\n",
        "with open(\"test_data.txt\", \"w\") as f:\n",
        "  f.writelines(test_data)"
      ],
      "metadata": {
        "id": "KGJ7Kd4hUOPP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gpt2_generator = pipeline('text-generation', model='gpt2', device=device)"
      ],
      "metadata": {
        "id": "n860kuDMQD90"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"gpt2\")"
      ],
      "metadata": {
        "id": "r1V_O6EDU-qi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = TextDataset(\n",
        "    tokenizer = tokenizer,\n",
        "    file_path = \"./train_data.txt\",\n",
        "    block_size = 64\n",
        ")\n",
        "\n",
        "test_dataset = TextDataset(\n",
        "    tokenizer = tokenizer,\n",
        "    file_path = \"./test_data.txt\",\n",
        "    block_size = 64\n",
        ")\n",
        "\n",
        "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bWJBNRqdSK-X",
        "outputId": "e20d0816-7105-4706-d465-2bdf284df937"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/data/datasets/language_modeling.py:53: FutureWarning: This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_mlm.py\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gpt2_model = AutoModelWithLMHead.from_pretrained(\"gpt2\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gnhqCEd8VmyC",
        "outputId": "2e3d529d-aad4-4590-aad6-96faac65c6fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/modeling_auto.py:1479: FutureWarning: The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use `AutoModelForCausalLM` for causal language models, `AutoModelForMaskedLM` for masked language models and `AutoModelForSeq2SeqLM` for encoder-decoder models.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir=\"./gpt2_chef\",\n",
        "    overwrite_output_dir=True,\n",
        "    num_train_epochs=1,\n",
        "    per_device_train_batch_size=32,\n",
        "    per_device_eval_batch_size=32,\n",
        "    eval_steps=100,\n",
        "    save_steps=500,\n",
        "    warmup_steps=1000\n",
        ")"
      ],
      "metadata": {
        "id": "XdLjRA4mWFoB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = Trainer(\n",
        "    model=gpt2_model,\n",
        "    args=training_args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=test_dataset\n",
        ")"
      ],
      "metadata": {
        "id": "RndMgfGsWrOg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 93
        },
        "id": "kWqmfztRW4pw",
        "outputId": "eb907dd9-8f02-475c-d931-404436e229ba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='88' max='3310' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [  88/3310 01:27 < 54:46, 0.98 it/s, Epoch 0.26/10]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.save_model()"
      ],
      "metadata": {
        "id": "sY4uyiYjXHh3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gpt2_chef = pipeline(\"text-generation\", model=\"./gpt2_chef\", tokenizer=\"gpt2\")"
      ],
      "metadata": {
        "id": "vpovevmHYpBj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gpt2_chef(\"The chicken\", max_length=1024)[0][\"generated_text\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "tFzo5-FfY6YD",
        "outputId": "647072c5-6615-43b9-8c45-0fb0beb91aba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The chicken pieces equation needs to be set properly.\\nKeep aside for 30 minutes.Now we will go to the kitchen with the chicken pieces and mash them first.Heat oil in a wok/kadhai and sieve into a wide pot (about 5 cm diameter).Add in the cumin seeds, turmeric powder, coriander powder, salt and ginger to the hot water and saute for a couple of minutes.Finally add the lentils, green chillies, garlic and turmeric powder and saute for a few minutes till aromatic.Add in the chopped coriander leaves, green chilli powder, red chilli powder and saute for 3 more minutes.Next add in the chopped ginger and saute for about 2 more minute.\\nAdd the boiled chickpeas and saute till the tadka is done.When the chickpeas are done add the turmeric and green chilli powder.Add the paneer seeds, salt and cumin seeds and saute till the tadka is cooked.\\nFinally add the coriander leaves, chopped coriander leaves, red chilli powder with a little water by adding water on medium heat, mix well and cook for 2 more minutes till the chickpeas are done.Once done, cook the chickpeas as per your taste.\\nServe the lentil-garlic chicken paratha along with the Pav Kala and the Panna Chutney or as a snack along with Chutney and Dosa Recipe (Kerala Style Spinach Salad).To begin the steaming process, firstly clean and dry the lentils and add green chilli seeds.\\nCookHillary Dosa Recipe till tender.Open the cooker and add the cooked khaniya along with Eating With Chicken and Chickpea Follies to the cooker.Pour the green chilli powder, coriander powder, garam masala and curry powder with the steamed lentil chickpeas and stir everything well well and allow them to cook.Check the salt and spices on the prepared Khanna Chutney and if you can add a bit of ground cumin seeds (such as red), it will help absorb the taste and aroma of tamarind and keep it aside.In a saucepan, heat a tablespoon of the curry powder over medium-low flame.Add the chopped garlic and saute until aromatic.Add in the cooked chickpeas, turmeric powder, capsicum powder and saute till the chickpeas do not sizzle.Add the cooked chickpeas in the same pan and saute for 2 more minutes.Add in the raw cumin seeds and cook till the chickpeas start sizzling.Add salt, curry powder and 1 tablespoon coconut water once the chickpeas are cooked for 30 seconds.After 30 seconds, add the cooked chickpeas in the pan and turn off the flame and serve hot.Serve the Kathai Khaniya Paati Paratha along with Thaniya Bhindi (Chickpea Balsamic) Recipe along with Masala Dal Recipe or Paratha Recipe.\\nCheck the salt and spices on the prepared khaniya and if you can add a bit of ground cumin seeds (such as red), it will help absorb the taste and aroma of tamarind and keep it aside.\\nTo begin making the Dhaniya Masala Dal recipe, in a pan, heat the oil in a wok/kadhai.Add the ginger, turmeric powder, coriander salt, lemon juice and saute for a few seconds till the ginger softens and leaves a soft green color.Add the cooked Dhaniya Masala Dal recipe, kadhai, tomatoes, green chillies and the coriander leaves.\\nSimmer and allow it to cook for 1 minute.Once done, remove the lid and allow the Dal to cook and take out the tamarind out.\\nTurn off the flame and serve.Serve Kathai Khaniya Paati Paratha along with Thaniya Bhindi recipe or Masala Dal Recipe along with Masala Dal Recipe or Punjabi Style Masala Dal Recipe.\\nTo begin the Steaming process, first clean and dry the lentils and salt and spices and add to the cooker along with soaking.\\nCook till the lentils and the spices are thickened to keep the Dal steamed.Once steamed turn off the flame and let the Dal cook for 5-10 minutes till the turmeric and black pepper turns soft.Take out the Dal soaked in water and roast it in the pan till it turns soft.Once done, add the cooked khaniya and coriander leaves, turmeric powder and stir through.We will stir it with little water to bring the Dal steamed to a soft consistency.\\nWhen done, take out the Dal steamed and roast it in hot water and allow to cool.Once the Dal steamed is steamed, add ginger and saute till the leaves turn'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gpt2_generator(\"The chicken\")[0][\"generated_text\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "id": "dtdDYjOVZBJ1",
        "outputId": "8c122727-11c9-4437-9d3e-c1264d0c267d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The chicken was the easiest,\" he said. By the time he went into the water, his father was sitting on the edge of a sink. \"I saw the water was boiling,\" he recalled. \"And my dad would be lying in the sink'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Qfkbjw_zZfXT"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}