Copy_of_textClassification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of textClassification.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3",
      "language": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/getmrinal/ML-Notebook/blob/master/Copy_of_textClassification.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "bOy0Yzvxko84",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from nltk.corpus import stopwords\n",
        "import os\n",
        "import re\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import time\n",
        "# from google.colab import files as filess"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "w-5j-1Aako89",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "15221f24-4f5f-4a99-f9a5-79120966524d"
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /content/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "metadata": {
        "id": "BSrm1Dw_ko9E",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "61ac9932-7b98-4a8c-80e2-99e321cc6f5a"
      },
      "cell_type": "code",
      "source": [
        "import urllib.request\n",
        "urllib.request.urlretrieve (\"https://archive.ics.uci.edu/ml/machine-learning-databases/20newsgroups-mld/20_newsgroups.tar.gz\", \"a.tar.gz\")\n",
        "urllib.request.urlretrieve (\"http://archive.ics.uci.edu/ml/machine-learning-databases/20newsgroups-mld/mini_newsgroups.tar.gz\", \"b.tar.gz\")"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('b.tar.gz', <http.client.HTTPMessage at 0x7f9fab7f71d0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "metadata": {
        "id": "mZdjZLIEko9J",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import tarfile\n",
        "tar = tarfile.open(\"a.tar.gz\")\n",
        "tar2 = tarfile.open(\"b.tar.gz\")\n",
        "tar.extractall()\n",
        "tar2.extractall()\n",
        "tar.close()\n",
        "tar2.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cI04t02Dko9N",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6Y5q55a9ko9Q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "1faedbed-c33e-4e61-8118-f5f7d7b0ee92"
      },
      "cell_type": "code",
      "source": [
        "stop_words = set(stopwords.words('english'))\n",
        "block_wrds = ['sender:','subject:','from:','date:','newsgroups:','message-id:','lines:','path:','re:','--','sender:','last','better','never','every','even','two','good','used','first','need','going','must','really','might','well','without','made','give','look','try','far','less','seem','new','make','many','way','since','using','take','help','thanks','send','free','may','see','much','want','find','would','one','like','get','use','also','could','say','us','go','please','said','set','got','sure','come','lot','seems','able','anything','put']\n",
        "\"writes\" in block_wrds"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "fl967rpjko9W",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "strip the documents and take into dictionary after the header only!"
      ]
    },
    {
      "metadata": {
        "id": "XsiIOY21ko9X",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dictionary = {}\n",
        "type(dictionary)\n",
        "count=0\n",
        "for file in os.listdir(\"mini_newsgroups\"):\n",
        "    for files in os.listdir(\"mini_newsgroups/\"+file):\n",
        "        #print(file,files)\n",
        "        f = open(\"mini_newsgroups/\"+file+\"/\"+files,'r',errors='ignore')\n",
        "        message = f.read()\n",
        "        message = message.split()\n",
        "        k =1\n",
        "        for i in message:\n",
        "            count +=1\n",
        "            if(len(i) > 1):\n",
        "                if not i.lower() in stop_words:\n",
        "                    if not i.lower() in block_wrds:\n",
        "                        if(i.lower() in dictionary.keys()):\n",
        "                            dictionary[i.lower()] = dictionary[i.lower()] +1\n",
        "                        else:\n",
        "                            dictionary[i.lower()] = 1\n",
        "\n",
        "\n",
        "        f.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7DqLUmzVko9a",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import operator\n",
        "sorted_vocab = sorted(dictionary.items(), key= operator.itemgetter(1), reverse= True)\n",
        "# sorted_vocab"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "z-5xaivUko9f",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "top_val = []\n",
        "sorted_vocab[1000][1]\n",
        "size = len(sorted_vocab)\n",
        "for i in range(size):\n",
        "    if(sorted_vocab[1000][1] <= sorted_vocab[i][1]):\n",
        "        top_val.append(sorted_vocab[i][0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9XKbg8Zmko9k",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1817
        },
        "outputId": "279e47be-f582-4d8a-be6d-ecd7f8178467"
      },
      "cell_type": "code",
      "source": [
        "top_val[0:100]"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['apr',\n",
              " 'organization:',\n",
              " 'gmt',\n",
              " '1993',\n",
              " 'writes:',\n",
              " 'references:',\n",
              " 'article',\n",
              " '|>',\n",
              " '>>',\n",
              " 'know',\n",
              " 'people',\n",
              " 'nntp-posting-host:',\n",
              " '93',\n",
              " 'university',\n",
              " 'think',\n",
              " 'xref:',\n",
              " 'cantaloupe.srv.cs.cmu.edu',\n",
              " \"i'm\",\n",
              " 'time',\n",
              " 'distribution:',\n",
              " 'it.',\n",
              " 'anyone',\n",
              " 'system',\n",
              " 'world',\n",
              " '25',\n",
              " 'still',\n",
              " 'image',\n",
              " 'computer',\n",
              " '20',\n",
              " 'problem',\n",
              " 'believe',\n",
              " 'data',\n",
              " \"i've\",\n",
              " '16',\n",
              " 'work',\n",
              " 'something',\n",
              " \"can't\",\n",
              " '>in',\n",
              " '>the',\n",
              " 'reply-to:',\n",
              " '21',\n",
              " \"that's\",\n",
              " 'state',\n",
              " 'news',\n",
              " 'another',\n",
              " 'back',\n",
              " 'right',\n",
              " '15',\n",
              " 'information',\n",
              " '23',\n",
              " 'windows',\n",
              " 'number',\n",
              " 'available',\n",
              " 'usa',\n",
              " 'program',\n",
              " 'government',\n",
              " '>i',\n",
              " '22',\n",
              " 'power',\n",
              " 'god',\n",
              " 'file',\n",
              " 'software',\n",
              " 'someone',\n",
              " 'least',\n",
              " 'tue,',\n",
              " 'part',\n",
              " 'point',\n",
              " 'space',\n",
              " 'fri,',\n",
              " 'run',\n",
              " 'probably',\n",
              " 'different',\n",
              " 'read',\n",
              " 'drive',\n",
              " 'things',\n",
              " 'years',\n",
              " '(usenet',\n",
              " 'it,',\n",
              " 'mr.',\n",
              " 'little',\n",
              " '19',\n",
              " '17',\n",
              " 'case',\n",
              " 'best',\n",
              " 'tell',\n",
              " 'public',\n",
              " 'enough',\n",
              " 'around',\n",
              " 'question',\n",
              " '10',\n",
              " 'long',\n",
              " 'mon,',\n",
              " 'however,',\n",
              " 'support',\n",
              " 'rather',\n",
              " 'access',\n",
              " 'thing',\n",
              " 'science',\n",
              " 'let',\n",
              " 'group']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "metadata": {
        "id": "NKN-U6wFko9q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "77897d3b-71a5-4641-87d2-bb05e681edf0"
      },
      "cell_type": "code",
      "source": [
        "df = pd.DataFrame(columns = top_val)\n",
        "df.columns\n",
        "start_time = time.time()\n",
        "start_time"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1527524703.9376035"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "metadata": {
        "id": "ijr8ig6Pko9u",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "70c29828-badb-4aed-c379-32038c30c791"
      },
      "cell_type": "code",
      "source": [
        "#@title Default title text\n",
        "df = pd.DataFrame(columns = top_val)\n",
        "df.columns\n",
        "count=0\n",
        "for file in os.listdir(\"20_newsgroups\"):\n",
        "    for files in os.listdir(\"20_newsgroups/\"+file):\n",
        "        count=count+1\n",
        "        #print(file,files)\n",
        "        df.loc[len(df)] = np.zeros(len(top_val))\n",
        "        f = open(\"20_newsgroups/\"+file+\"/\"+files,'r',errors='ignore')\n",
        "        message = f.read()\n",
        "        message = message.split()\n",
        "        k =0\n",
        "        for i in message:\n",
        "            if(i.lower() in df.columns):\n",
        "                df[i.lower()][len(df)-1] += 1\n",
        "        f.close()\n",
        "count"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "19997"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "metadata": {
        "id": "hiEEemn9ko90",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "78e3ddc1-ae0f-4905-e881-bef82d5ab337"
      },
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(19997, 1018)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "metadata": {
        "id": "wLOg4Tm6ko95",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "y=[]\n",
        "i=0\n",
        "count=0\n",
        "for file in os.listdir(\"20_newsgroups\"):\n",
        "    for files in os.listdir(\"20_newsgroups/\"+file):\n",
        "        #print(file,files)\n",
        "        count+=1\n",
        "        y.append(i)\n",
        "    i=i+1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "r--3MW5bko9-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "2dd8d944-e76e-44c5-df31-9e6b49d81516"
      },
      "cell_type": "code",
      "source": [
        "y = np.array(y)\n",
        "y.shape,df.shape\n",
        "x = df.values\n",
        "count"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "19997"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "metadata": {
        "id": "uXN1658wko-D",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn import model_selection\n",
        "x_train, x_test, y_train, y_test = model_selection.train_test_split(x, y, test_size = 0.25, random_state = 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1AaYbzniko-H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "10a51b02-7938-4cfd-d2d0-62b53dbedd41"
      },
      "cell_type": "code",
      "source": [
        "from sklearn.naive_bayes import MultinomialNB\n",
        "clf = MultinomialNB()\n",
        "clf.fit(x_train, y_train)\n",
        "y_pred = clf.predict(x_test)\n",
        "\n",
        "train_score = clf.score(x_train, y_train)\n",
        "test_score = clf.score(x_test, y_test)\n",
        "\n",
        "train_score, test_score"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.8495699139827966, 0.8116)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "metadata": {
        "id": "jysYIi8wko-K",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "8ebadf8e-ef6b-4ad2-9a5a-861a3685b8c4"
      },
      "cell_type": "code",
      "source": [
        "newData = df\n",
        "newData['out'] = y\n",
        "newData.to_csv(\"textClassification.csv\")\n",
        "end_time = time.time()\n",
        "total_time = end_time - start_time\n",
        "total_time"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "981.5265352725983"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "metadata": {
        "id": "WgGRXdwhjcff",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "filess.download('textClassification.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1Nx2pT1qko-O",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "data = pd.read_csv(\"textClassification.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dt3lmd3Fh_nJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "fc9be360-8e9f-4cfc-9af5-e34d11c23b3b"
      },
      "cell_type": "code",
      "source": [
        "data = pd.read_csv(\"textClassification.csv\")\n",
        "Y = data[\"out\"]\n",
        "print(data.shape)\n",
        "data.drop(['out'], axis = 1, inplace = True)\n",
        "data.drop(['Unnamed: 0'], axis = 1, inplace = True)\n",
        "\n",
        "print(data.shape)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(19997, 1020)\n",
            "(19997, 1018)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "t5zx62gRiB2l",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn import model_selection\n",
        "X_train, X_test, Y_train, Y_test = model_selection.train_test_split(data, Y, test_size = 0.30,shuffle=True, random_state = 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LsDt07qKiHLf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c3632f9b-8117-4ce5-f90d-df0b9f93783d"
      },
      "cell_type": "code",
      "source": [
        "f_list = list(data)\n",
        "xx = X_train[Y_train == 0]\n",
        "xx.shape,Y_train.shape,X_train.shape,X_test.shape"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((726, 1018), (13997,), (13997, 1018), (6000, 1018))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "metadata": {
        "id": "i3DOxp0PiKY5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def train(X_train,Y_train):\n",
        "    result = {}\n",
        "    set_class = set(Y_train)\n",
        "    for curr_class in set_class:\n",
        "        result[curr_class] = {}\n",
        "        result[\"total_data\"] = len(Y_train)\n",
        "        #all the x_train rows whose Y is curr_class\n",
        "        curr_class_rows = (Y_train == curr_class)\n",
        "        X_train_curr = X_train[curr_class_rows]\n",
        "        Y_train_curr = Y_train[curr_class_rows]\n",
        "        #traverse through all the features of X_train and get the sum of each word and save it in the dict\n",
        "        #i.e result[class][word] = count of word appearance in the doc\n",
        "        sums = 0\n",
        "        for x in f_list:\n",
        "            result[curr_class][x] = X_train_curr[x].sum()\n",
        "            sums = sums+result[curr_class][x]\n",
        "        result[curr_class][\"total_count\"] = sums\n",
        "    return result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PiZdSD2xiNuV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "4334c833-54a0-4288-fead-bae709e429ac"
      },
      "cell_type": "code",
      "source": [
        "dictionary = train(X_train,Y_train)\n",
        "len(dictionary[0]),len(f_list)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1019, 1018)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "metadata": {
        "id": "5jckKGVmiS-b",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def probablity(dictionary,x,current_class):\n",
        "    output= np.log(dictionary[current_class][\"total_count\"])-np.log(dictionary[\"total_data\"])\n",
        "   # num_features = len(dictionary[current_class].keys())-1;\n",
        "    for j in f_list:\n",
        "        count_current_class_with_word_i = dictionary[current_class][j] + 1 \n",
        "        count_current_class = dictionary[current_class][\"total_count\"] + len(f_list)\n",
        "        current_xj_prob = np.log(count_current_class_with_word_i) -np.log(count_current_class)\n",
        "        output = output + current_xj_prob\n",
        "    return output "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hkul8lXbiVot",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def predictSinglePoint(dictionary,x):\n",
        "    classes = dictionary.keys()\n",
        "    best_p = -1000\n",
        "    best_class = -1\n",
        "    first_run = True\n",
        "    for current_class in classes:\n",
        "        if(current_class == \"total_data\"):\n",
        "            continue\n",
        "        p_curr_class = probablity(dictionary,x,current_class)\n",
        "        if(first_run or p_curr_class > best_p):\n",
        "            best_p = p_curr_class\n",
        "            best_class = current_class\n",
        "        first_run = False\n",
        "    return best_class"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Y7Y5DuH9iZTg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def predict(dictionary,X_test):\n",
        "    Y_pred = []\n",
        "    for j in range(len(X_test)):\n",
        "        \n",
        "        x_class = predictSinglePoint(dictionary,x)\n",
        "        Y_pred.append(x_class)\n",
        "    return Y_pred"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vx-X1xwTidZv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        " Y_pred = predict(dictionary,X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "STL5zDG2igtN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "48a2cce6-9f5e-4425-d88f-eadfce7fd093"
      },
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "accuracy_score(Y_test,Y_pred)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.051333333333333335"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    }
  ]
}