1_Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "da9b3225-41ff-4eca-b364-9b2f83b976db",
   "metadata": {},
   "source": [
    "# End to end process for adding both entity ruler and word vectors to an NER model\n",
    "1. Document cleaning and splitting the corpus into test and train sets\n",
    "2. Build word vectors\n",
    "3. Build training data with entity ruler and split into train and validation data\n",
    "4. Add word vectors to model, run"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e507f17-b2f3-4e2b-94aa-61f334b38379",
   "metadata": {},
   "source": [
    "## Notebook 1\n",
    "- Load the documents\n",
    "- Segment documents into sentences and strip basic punctuation\n",
    "- Remove additional white spaces, punctuation, and stop words\n",
    "- Shuffle the corpus\n",
    "- Extract hold out data for testing\n",
    "- Save training and testing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "76f0588e-2076-440e-965f-1cf64b015bee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/sarasharick/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/sarasharick/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "#import files\n",
    "import glob\n",
    "import docx2txt\n",
    "\n",
    "#splitting into sentences\n",
    "import spacy\n",
    "from spacy.lang.en import English\n",
    "\n",
    "#cleaning\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "from nltk.tokenize import word_tokenize\n",
    "from gensim.parsing.preprocessing import strip_punctuation\n",
    "from gensim.parsing.preprocessing import strip_multiple_whitespaces\n",
    "import contractions\n",
    "\n",
    "#shuffle corpus\n",
    "import random\n",
    "\n",
    "#save data\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62f442d9-ab16-4bd9-a71f-5772e1cc8e3a",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4ef45f59-bc15-429b-b98b-c54c93c4bed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import files\n",
    "files = []\n",
    "for file in glob.glob('./docs/*.docx'):\n",
    "    files.append(docx2txt.process(file))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d58d66e-4eea-4fba-a05b-b1936f895e29",
   "metadata": {},
   "source": [
    "### Split into sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fbcc98f4-2fbf-4140-832d-6e7e2fd08edd",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = English() \n",
    "nlp.add_pipe('sentencizer') #other sentence splitters seemed to have trouble with question marks\n",
    "\n",
    "\n",
    "corpus = []\n",
    "for file in files:\n",
    "    file = file.replace('“', '').replace('”', '') #removes quotation marks\n",
    "    doc = nlp(file)\n",
    "    for sent in doc.sents:\n",
    "        str(sent).strip()\n",
    "        corpus.append(sent.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7dcc978c-53c6-4a22-be27-270e06ad3e0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It was a nice section of Coruscant, high up, near lots of shopping and entertainment, but also easily accessible from Starfighter Command headquarters.\n"
     ]
    }
   ],
   "source": [
    "#review corpus to ensure it came out correctly\n",
    "print(corpus[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88743804-6b0a-401b-9cb3-06d5496345f0",
   "metadata": {},
   "source": [
    "### Prepare stop words and clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "63f5cf9d-6e74-4183-b569-5b70c75bd461",
   "metadata": {},
   "outputs": [],
   "source": [
    "#prep stop words list with capitals\n",
    "stops = stopwords.words('English')\n",
    "add_stops = [word.capitalize() for word in stopwords.words('English')]\n",
    "stops.extend(add_stops)\n",
    "more_stops = ['said', 'would', 'could', 'back', 'oh', 'Oh', 'Well', 'like', 'around', 'time', 'one', 'get',\n",
    "              'to', 'know', 'us' , 'got', 'Um', 'um', 'Look']\n",
    "stops.extend(more_stops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a2bac56c-ce44-4999-9e0f-c79d978a0232",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_clean = []\n",
    "for sentence in corpus:\n",
    "    sentence = contractions.fix(sentence)  #expand contractions\n",
    "    sentence = sentence.replace(\"’s\", '').replace(\"'s\", '').replace(\"‘s\", '') #remove possessives\n",
    "    sentence = sentence.replace(\"'\", '').replace(\"’\", '').replace(\"‘\", '').replace(\"——\", '') #remove excess aopstrophes in all forms\n",
    "    sentence = strip_punctuation(sentence) #strip additional punctuation\n",
    "    sentence = word_tokenize(sentence) \n",
    "    sentence = [word for word in sentence if word not in stops]\n",
    "    sentence = ' '.join(sentence)\n",
    "    sentence = strip_multiple_whitespaces(sentence) #strip linebreaks\n",
    "    sentence = sentence.strip() #strip any additional stray whitespace\n",
    "    corpus_clean.append(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b22a42cb-d3b5-4475-a0a6-8d7128144fce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nice section Coruscant high near lots shopping entertainment also easily accessible Starfighter Command headquarters\n"
     ]
    }
   ],
   "source": [
    "#review corpus again\n",
    "print(corpus_clean[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69c92335-4429-408c-b8cc-76751311cc31",
   "metadata": {},
   "source": [
    "### Shuffle corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1820be4b-5f9d-49d0-b511-a4b104594e6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle corpus\n",
    "random.shuffle(corpus_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "78564def-a1f2-4774-9909-e4c3767368ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18399\n"
     ]
    }
   ],
   "source": [
    "print(len(corpus_clean))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f60825f5-3a2b-445d-9166-cc8309200216",
   "metadata": {},
   "source": [
    "### Split corpus into training and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "68fcc720-dbfb-46bc-a9a4-b0ac3788a40a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unlabeled sentences for testing:  1839\n",
      "Sentences to be labeled for training and validation:  16560\n"
     ]
    }
   ],
   "source": [
    "#extract 10% hold out test data\n",
    "test_corpus = corpus_clean[-1839:]\n",
    "print('Unlabeled sentences for testing: ', len(test_corpus))\n",
    "corpus_clean = corpus_clean[:-1839]\n",
    "print('Sentences to be labeled for training and validation: ', len(corpus_clean))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22272342-6cf7-4362-ab6b-dcf2232e4337",
   "metadata": {},
   "source": [
    "### Save training and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8b1dc56b-7134-40bf-9239-c292f31dcf3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function to save data as json file\n",
    "def save_data(file, data):\n",
    "    with open(file, 'w', encoding='utf-8') as f:\n",
    "        json.dump(data, f, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5aabbaa3-77a8-45c0-8bdf-28b2cc9b0810",
   "metadata": {},
   "outputs": [],
   "source": [
    "#save hold out test data\n",
    "save_data('data/sw_test_ner.json', test_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0b19e9de-e58d-49bc-a7bc-878edeec4be6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#save training data\n",
    "save_data('data/sw_train_ner.json', corpus_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e46b39fc-5ad7-4d9d-97b7-bef79b1390f2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}