complete-html-or-pdf-to-txt_CorpusWriter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "from urllib import request\n",
    "from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter\n",
    "from pdfminer.converter import TextConverter\n",
    "from pdfminer.layout import LAParams\n",
    "from pdfminer.pdfpage import PDFPage\n",
    "import re\n",
    "import urllib\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# link-to-pdf downloader:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('download.pdf', <http.client.HTTPMessage object at 0x000002637D66D3C8>)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# the line below is for pullling a pdf from a link and downloading it to your python path folder. the pdf will be later converted to text and then modified and written to a text corpus.\n",
    "request.urlretrieve(\"http://www.rollanet.org/~n0klu/Ham_Radio/(eBook)%20Electronics%20-%20The%20Electrical%20Engineering%20Handbook.pdf\", \"download.pdf\")\n",
    "## the 2 arguments given are (\"direct link to pdf\", \"what you want to name the download (e.g. \"download.pdf\", \"download13.pdf\", \"Fundamentals-of-Electronics.pdf\", etc.)\")\n",
    "\n",
    "# this step is not necessary if you download a pdf manually annd put it in the python path folder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pdf-to-.txt converter: function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def pdf_to_txt(path):\n",
    "    rsrcmgr = PDFResourceManager()\n",
    "    retstr = io.StringIO()\n",
    "    codec = 'utf-8'\n",
    "    laparams = LAParams()\n",
    "    device = TextConverter(rsrcmgr, retstr, laparams=laparams, imagewriter=None)\n",
    "    fp = open(path, 'rb')\n",
    "    interpreter = PDFPageInterpreter(rsrcmgr, device)\n",
    "    password = \"\"\n",
    "    maxpages = 0\n",
    "    caching = True\n",
    "    pagenos=set()\n",
    "    for page in PDFPage.get_pages(fp, pagenos, maxpages=maxpages,password=password,caching=caching, check_extractable=True):\n",
    "        interpreter.process_page(page)\n",
    "\n",
    "    text = retstr.getvalue()\n",
    "\n",
    "    fp.close()\n",
    "    device.close()\n",
    "    retstr.close()\n",
    "    return text\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pdf-to-.txt converter: convert file and clean texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      "\f",
      " \n",
      "\n",
      " \n",
      "\n",
      "WAR AND PEACE \n",
      "\n",
      " \n",
      " \n",
      "BY  \n",
      "\n",
      "LEO TOLSTOY \n",
      "\n",
      " \n",
      "\n",
      "TRANSLATED BY LOUISE AND AYLMER MAU\n"
     ]
    }
   ],
   "source": [
    "text = pdf_to_txt(\"war-and-peace.pdf\") # again, the name of the pdf that you want to convert to text\n",
    "print(text[:100])\n",
    "\n",
    "#text = convert_pdf_to_txt(\"war-and-peace.pdf\")    \n",
    "\n",
    "listt = []\n",
    "for i in text.split('\\n\\n'):\n",
    "    listt.append(i)\n",
    "listylistt = []\n",
    "listylisttlen = []\n",
    "for i in listt:\n",
    "    listylisttlen.append(i)\n",
    "    listylistt.append([i])\n",
    "\n",
    "\n",
    "listycleaned = []\n",
    "for i in range(len(listylistt)):                   #getting rid of unwanted short paragraphs\n",
    "    if len(listylisttlen[i]) > 50:\n",
    "        listycleaned.append(listylistt[i])    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# identify where table of contents and body text end:\n",
    "\n",
    "modify the writer to only write snippets in the body text to the corpus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['Chapter 11 \\nChapter 12 \\nChapter 13 \\nChapter 14 \\nChapter 15 \\nChapter 16 \\nChapter 17 \\nChapter 18 \\nChapter 19 \\nChapter 20 \\nChapter 21 \\nChapter 22 \\nChapter 23 \\nChapter 24 \\nChapter 25 \\nChapter 26 \\nChapter 27 \\nChapter 28 \\nChapter 29 \\nChapter 30 \\nChapter 31 \\nChapter 32 \\nChapter 33 \\nChapter 34 \\nBOOK 12. 1812 '], ['Chapter 4 \\nChapter 5 \\nChapter 6 \\nChapter 7 \\nChapter 8 \\nChapter 9 \\nChapter 10 \\nChapter 11 \\nChapter 12 \\nChapter 13 \\nChapter 14 \\nChapter 15 \\nChapter 16 \\nBOOK 13. 1812 '], ['Chapter 1 \\nChapter 2 \\nChapter 3 \\nChapter 4 \\nChapter 5 \\nChapter 6 \\nChapter 7 \\nChapter 8 \\nChapter 9 \\nChapter 10 \\nChapter 11 \\nChapter 12 \\nChapter 13 \\nChapter 14 '], ['Chapter 15 \\nChapter 16 \\nChapter 17 \\nChapter 18 \\nChapter 19 \\nBOOK 14. 1812 '], ['Chapter 1 \\nChapter 2 \\nChapter 3 \\nChapter 4 \\nChapter 5 \\nChapter 6 \\nChapter 7 \\nChapter 8 \\nChapter 9 \\nChapter 10 \\nChapter 11 \\nChapter 12 \\nChapter 13 \\nChapter 14 \\nChapter 15 \\nChapter 16 \\nChapter 17 \\nChapter 18 \\nChapter 19 '], ['Chapter 3 \\nChapter 4 \\nChapter 5 \\nChapter 6 \\nChapter 7 \\nChapter 8 \\nChapter 9 \\nChapter 10 \\nChapter 11 \\nChapter 12 \\nChapter 13 \\nChapter 14 \\nChapter 15 \\nChapter 16 \\nChapter 17 \\nChapter 18 \\nChapter 19 \\nChapter 20 '], ['Chapter 1 \\nChapter 2 \\nChapter 3 \\nChapter 4 \\nChapter 5 \\nChapter 6 \\nChapter 7 \\nChapter 8 \\nChapter 9 '], ['Chapter 10 \\nChapter 11 \\nChapter 12 \\nChapter 13 \\nChapter 14 \\nChapter 15 \\nChapter 16 '], ['Chapter 1 \\nChapter 2 \\nChapter 3 \\nChapter 4 \\nChapter 5 \\nChapter 6 \\nChapter 7 \\nChapter 8 \\nChapter 9 \\nChapter 10 \\nChapter 11 \\nChapter 12 '], ['\"Well, Prince, so Genoa and Lucca are now just family estates of the \\nBuonapartes. But I warn you, if you don\\'t tell me that this means war, if you \\nstill try to defend the infamies and horrors perpetrated by that Antichrist—I \\nreally believe he is Antichrist—I will have nothing more to do with you and \\nyou are no longer my friend, no longer my \\'faithful slave,\\' as you call \\nyourself! But how do you do? I see I have frightened you—sit down and tell \\nme all the news.\" '], ['It was in July, 1805, and the speaker was the well-known Anna Pavlovna \\nScherer, maid of honor and favorite of the Empress Marya Fedorovna. With \\nthese words she greeted Prince Vasili Kuragin, a man of high rank and \\nimportance, who was the first to arrive at her reception. Anna Pavlovna had \\nhad a cough for some days. She was, as she said, suffering from la grippe; \\ngrippe being then a new word in St. Petersburg, used only by the elite. '], ['All her invitations without exception, written in French, and delivered by a \\nscarlet-liveried footman that morning, ran as follows: ']]\n",
      "['\"Well, Prince, so Genoa and Lucca are now just family estates of the \\nBuonapartes. But I warn you, if you don\\'t tell me that this means war, if you \\nstill try to defend the infamies and horrors perpetrated by that Antichrist—I \\nreally believe he is Antichrist—I will have nothing more to do with you and \\nyou are no longer my friend, no longer my \\'faithful slave,\\' as you call \\nyourself! But how do you do? I see I have frightened you—sit down and tell \\nme all the news.\" ']\n",
      "10804\n"
     ]
    }
   ],
   "source": [
    "print(listycleaned[15:27])\n",
    "print(listycleaned[24])\n",
    "print(len(listycleaned))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# calculate for i and range, write corpus "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#where i is the beginning of the body text, and where range is end of body text minus i: \n",
    "#                                here we say i=25(beginning of body text), and range(x)= 10779 (10804-25)        \n",
    "i = 25       \n",
    "for j in range(10779):\n",
    "    for value in listycleaned[i]:\n",
    "        i += 1\n",
    "        f = io.open(\"foldy/TestCorpusWriter/snippet01_%s.txt\" % i, 'w', encoding=\"utf-8\")       ## change chunk name ('snippet01_01' becomes 'snippet01_02')\n",
    "        f.write(value)\n",
    "        f.close()\n",
    "        \n",
    "        \n",
    "       "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # HTML-to-text corpus\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### pull html-to-raw-text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "589638"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = \"https://olney.ai/neets-web/Mod01%20-%20Matter%20Energy%20and%20DC.pdf-extracted/Mod01%20-%20Matter%20Energy%20and%20DC.pdf.xhtml-pretty.html\"\n",
    "html = request.urlopen(url).read().decode('utf8')\n",
    "response = request.urlopen(url)\n",
    "raw = response.read().decode('utf8')\n",
    "splits = BeautifulSoup(html, 'html.parser').find_all('p')\n",
    "len(raw)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pull raw text and file into lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "listy= []\n",
    "with open(\"paragraphsplit.txt\",\"wb\") as outfile:\n",
    "    for i in splits:\n",
    "        outfile.write(bytes(i.text+'\\n', 'UTF-8'))\n",
    "        listy.append(i.text)\n",
    "        \n",
    "listycleaned = []\n",
    "for i in range(len(listy)):                   #getting rid of unwanted short paragraphs\n",
    "    if len(listy[i]) > 140:\n",
    "        listycleaned.append(listy[i])\n",
    "        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## identify body text range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Electrostatics (electricity at rest) is a subject with which most persons entering the field of electricity\n",
      "and electronics are somewhat familiar. For example, the way a person’s hair stands on end after a\n",
      "vigorous rubbing is an effect of electrostatics. While pursuing the study of electrostatics, you will gain a\n",
      "better understanding of this common occurrence. Of even greater significance, the study of electrostatics\n",
      "will provide you with the opportunity to gain important background knowledge and to develop concepts\n",
      "which are essential to the understanding of electricity and electronics.\n",
      "\n",
      "815\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(listycleaned[90])\n",
    "print(len(listycleaned))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## write modified text lists to corpus     \n",
    "(don't forget to account for body-text range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(len([listycleaned])):\n",
    "    i = 0\n",
    "    for value in listycleaned:\n",
    "        i += 1\n",
    "        f = io.open(\"foldy/TestCorpusWriter/mod01_%s.txt\" % i, 'w', encoding=\"utf-8\")       ## change volume name ('mod01' becomes 'mod02'), chunk name is 'mod01_01', 'mod01_02', etc.\n",
    "        f.write(value)\n",
    "        f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you should now have a corpus of the chunked texts from the book in your TestCorpusWriter folder. you can add to the to the same corpus by repeating the script with a different link or pdf and renaming the file in *calculate for i and write corpus* cell (e.g. \"snippet\"). you can create per-document corpora by repeating the script with a new subfolfer name (TestCorpusWriter), *and* a new file name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "xpython",
   "language": "python",
   "name": "xpython"
  },
  "language_info": {
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}