notebook/NER_BERT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random \n",
    "from transformers import pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"../data/display_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Jaideep Prashar\\Desktop\\potential projects\\news-articles\\virtualenv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "c:\\Users\\Jaideep Prashar\\Desktop\\potential projects\\news-articles\\virtualenv\\Lib\\site-packages\\huggingface_hub\\file_download.py:157: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\Jaideep Prashar\\.cache\\huggingface\\hub\\models--dbmdz--bert-large-cased-finetuned-conll03-english. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n",
      "Some weights of the model checkpoint at dbmdz/bert-large-cased-finetuned-conll03-english were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                             content  \\\n",
      "0  DU Admissions 2023: The Delhi University today...   \n",
      "1  After winning the bronze medal in the men’s 25...   \n",
      "2  Some footballing inevitabilities of the New Ye...   \n",
      "3  Playing his 100th Test, Cheteshwar Pujara on S...   \n",
      "4  Reliance Industries chairman Mukesh Ambani ove...   \n",
      "\n",
      "                                            entities  \n",
      "0  [{'entity': 'I-ORG', 'score': 0.9958222, 'inde...  \n",
      "1  [{'entity': 'I-MISC', 'score': 0.79464215, 'in...  \n",
      "2  [{'entity': 'I-MISC', 'score': 0.66002136, 'in...  \n",
      "3  [{'entity': 'I-PER', 'score': 0.9996457, 'inde...  \n",
      "4  [{'entity': 'I-ORG', 'score': 0.99975556, 'ind...  \n"
     ]
    }
   ],
   "source": [
    "# Load the pre-trained NER model and tokenizer\n",
    "ner_pipeline = pipeline('ner', model='dbmdz/bert-large-cased-finetuned-conll03-english')\n",
    "\n",
    "# Define a function to perform NER on a single text\n",
    "def perform_ner(text):\n",
    "    return ner_pipeline(text)\n",
    "\n",
    "# Apply the NER function to the 'headline' column\n",
    "df['entities'] = df['content'].apply(perform_ner)\n",
    "\n",
    "# Display the dataframe with entities\n",
    "print(df[['content', 'entities']].head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"../data/entity_bert.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>category</th>\n",
       "      <th>entities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DU Admissions 2023: The Delhi University today...</td>\n",
       "      <td>education</td>\n",
       "      <td>[{'entity': 'I-ORG', 'score': 0.9958222, 'inde...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>After winning the bronze medal in the men’s 25...</td>\n",
       "      <td>sports</td>\n",
       "      <td>[{'entity': 'I-MISC', 'score': 0.79464215, 'in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Some footballing inevitabilities of the New Ye...</td>\n",
       "      <td>sports</td>\n",
       "      <td>[{'entity': 'I-MISC', 'score': 0.66002136, 'in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Playing his 100th Test, Cheteshwar Pujara on S...</td>\n",
       "      <td>sports</td>\n",
       "      <td>[{'entity': 'I-PER', 'score': 0.9996457, 'inde...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Reliance Industries chairman Mukesh Ambani ove...</td>\n",
       "      <td>business</td>\n",
       "      <td>[{'entity': 'I-ORG', 'score': 0.99975556, 'ind...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             content   category  \\\n",
       "0  DU Admissions 2023: The Delhi University today...  education   \n",
       "1  After winning the bronze medal in the men’s 25...     sports   \n",
       "2  Some footballing inevitabilities of the New Ye...     sports   \n",
       "3  Playing his 100th Test, Cheteshwar Pujara on S...     sports   \n",
       "4  Reliance Industries chairman Mukesh Ambani ove...   business   \n",
       "\n",
       "                                            entities  \n",
       "0  [{'entity': 'I-ORG', 'score': 0.9958222, 'inde...  \n",
       "1  [{'entity': 'I-MISC', 'score': 0.79464215, 'in...  \n",
       "2  [{'entity': 'I-MISC', 'score': 0.66002136, 'in...  \n",
       "3  [{'entity': 'I-PER', 'score': 0.9996457, 'inde...  \n",
       "4  [{'entity': 'I-ORG', 'score': 0.99975556, 'ind...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to extract entities and their respective scores from the entities column\n",
    "def extract_entities(entities):\n",
    "    extracted_entities = [(entity['entity'], entity['word'], entity['score']) for entity in entities]\n",
    "    return extracted_entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Observation:\n",
      "Content: The government may soon tweak the production-linked incentive scheme for pharmaceuticals, drones and textiles sectors to encourage investment and boost manufacturing, an official said.\n",
      "These sectors have been identified after inter-ministerial consultations on the performance of the scheme for various products.\n",
      "The official also said that disbursement of production-linked incentives (PLI) for white goods (AC and LED lights) would start this month and that would push the amount of disbursement, which was only Rs 2,900 crore till March 2023.\n",
      "ADVERTISEMENT\n",
      "The scheme was announced in 2021 for 14 sectors such as telecommunications, white goods, textiles, manufacturing of medical devices, automobiles, speciality steel, food products, high-efficiency solar PV modules, advanced chemistry cell battery, drones and pharma with an outlay of Rs 1.97 lakh crore.\n",
      "“We have identified the sectors. We are going to send the combined note to seek approval of the Union Cabinet. The changes include extending some time (for pharma sector) and adding some additional products in some sectors.\n",
      "“In textiles, we are expanding the definition of certain other products in the technical textiles segment; in drones, we are increasing the amount,” the government official, who did not wish to be named said.\n",
      "The total amount allocated for the PLI scheme for drones and drone components is Rs 120 crore spread over three financial years.\n",
      "ADVERTISEMENT\n",
      "A senior official of the commerce and industry ministry had earlier stated that some course corrections or tweaking were needed in PLI schemes that were not doing well.\n",
      "PLI schemes for sectors which are not picking up well include high-efficiency solar PV modules, advanced chemistry cell (ACC) batteries, textile products and speciality steel.\n",
      "On the other hand, the scheme is doing well in sectors such as electronics, pharma, medical devices, telecom, food processing, and white goods.\n",
      "ADVERTISEMENT\n",
      "The government is trying to sort out issues such as timely processing of claims, visa-related matters where vendors require Chinese professionals’ expertise, and delay in getting environmental clearances that have been raised by certain stakeholders of PLI schemes.\n",
      "The purpose of the scheme is to attract investments in key sectors and cutting-edge technology; ensure efficiency and bring economies of size and scale in the manufacturing sector; and make Indian companies and manufacturers globally competitive.\n",
      "These schemes for all 14 sectors have been notified by the concerned ministries/departments after due approval. These schemes are in various stages of implementation.\n",
      "The government is expected to disburse about Rs 13,000 crore to eligible firms seeking benefits under the schemes.\n",
      "Entities:\n",
      "- Type: I-ORG\n",
      "  Name: Union\n",
      "  Score: 0.9808106\n",
      "- Type: I-ORG\n",
      "  Name: Cabinet\n",
      "  Score: 0.95733666\n",
      "- Type: I-MISC\n",
      "  Name: Chinese\n",
      "  Score: 0.9978537\n",
      "- Type: I-MISC\n",
      "  Name: Indian\n",
      "  Score: 0.9985299\n"
     ]
    }
   ],
   "source": [
    "random_index = random.randint(0, len(df) - 1)\n",
    "\n",
    "# Extract entities for the random observation\n",
    "random_observation = df.iloc[random_index]\n",
    "extracted_entities = extract_entities(random_observation['entities'])\n",
    "\n",
    "# Print the output\n",
    "print(\"Random Observation:\")\n",
    "print(\"Content:\", random_observation['content'])\n",
    "print(\"Entities:\")\n",
    "for entity in extracted_entities:\n",
    "    print(\"- Type:\", entity[0])\n",
    "    print(\"  Name:\", entity[1])\n",
    "    print(\"  Score:\", entity[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('../data/entity_bert.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}