From 4dc5929b2928790a63125a8cec94171edaef7a02 Mon Sep 17 00:00:00 2001 From: Quanghuy99 <30777550+Quanghuy99@users.noreply.github.com> Date: Sat, 14 Aug 2021 19:29:10 +0700 Subject: [PATCH 1/2] Created using Colaboratory --- Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb | 3408 ++++++++++++++++++++++++ 1 file changed, 3408 insertions(+) create mode 100644 Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb diff --git a/Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb b/Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb new file mode 100644 index 0000000..01d3294 --- /dev/null +++ b/Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb @@ -0,0 +1,3408 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "accelerator": "TPU", + "colab": { + "name": "Copy of HACKANONS COLAB 25GB RAM.ipynb", + "provenance": [], + "collapsed_sections": [ + "4cbgwZWWfWpp" + ], + "machine_shape": "hm", + "include_colab_link": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "EN-xy98G3C95", + "outputId": "14450904-f512-483e-dd4e-e9ce341c1475" + }, + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive')" + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Mounted at /content/drive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "tAb77yZ9fzMG" + }, + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "%matplotlib inline" + ], + "execution_count": 2, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4cbgwZWWfWpp" + }, + "source": [ + "# preprocessing" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "pafL7Li0jyXW" + }, + "source": [ + "df = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/IoT-23/data_input/export_dataframe.csv',error_bad_lines=False,low_memory=False)" + ], + "execution_count": 3, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "RsRTAtuf3Mho" + }, + "source": [ + "pd.set_option('display.max_columns', None)" + ], + "execution_count": 4, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 507 + }, + "id": "8Gva_D-c3wQa", + "outputId": "cc3a868b-7d13-4425-acc5-ec1e9f8e483d" + }, + "source": [ + " df" + ], + "execution_count": 5, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#fieldstsuidid.orig_hid.orig_pid.resp_hid.resp_pprotoservicedurationorig_bytesresp_bytesconn_statelocal_origlocal_respmissed_byteshistoryorig_pktsorig_ip_bytesresp_pktsresp_ip_bytestunnel_parents label detailed-labellabel
0#typestimestringaddrportaddrportenumstringintervalcountcountstringboolboolcountstringcountcountcountcountset[string] string string0
11562159819.172195CUg3u41D8SwgQw1Job0.0.0.068255.255.255.25567udpdhcp30.00464287680S0--0D16921600- benign -NaN0
21562159849.173340CUpk9Y381SJuOHyK2d255.255.255.25568192.168.1.167udpdhcp0.00456403900SHR--0^d00134264- benign -NaN0
31562159953.959669CC5CQA3Ptzma7a0by4fe80::5bcc:698e:39d5:cdf5353ff02::fb5353udpdns3.9485398760S0--0D6116400- benign -NaN0
41562159998.302954C3sgcmfvNzjNoY0Kdfe80::5bcc:698e:39d5:cdf5353ff02::fb5353udpdns3.7681798760S0--0D6116400- benign -NaN0
........................................................................
178685201537542560.979740Ce0LkH33VlZBF8Vw93fe80::106c:9e5b:3af8:9cf143ff02::160icmp-1.945701400OTH--0-215200- benign -NaN0
178685211537542535.379473CdWSpn4EA2vXfUBqI7fe80::183b:7564:dbcc:3eca143ff02::160icmp-1.024087400OTH--0-215200- benign -NaN0
178685221537542504.559192ClKCqX3JOxjrti9lLhfe80::1847:a1bd:8d13:f43c143ff02::160icmp-59.390351800OTH--0-430400- benign -NaN0
178685231537542573.267978CDsdf81FPzUzFBBRBefe80::1c5c:213d:7cf8:890143ff02::160icmp----OTH--0-17600- benign -NaN0
17868524#close2019-01-03-20-02-05 benign -NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0
\n", + "

17868525 rows × 23 columns

\n", + "
" + ], + "text/plain": [ + " #fields ts \\\n", + "0 #types time \n", + "1 1562159819.172195 CUg3u41D8SwgQw1Job \n", + "2 1562159849.173340 CUpk9Y381SJuOHyK2d \n", + "3 1562159953.959669 CC5CQA3Ptzma7a0by4 \n", + "4 1562159998.302954 C3sgcmfvNzjNoY0Kd \n", + "... ... ... \n", + "17868520 1537542560.979740 Ce0LkH33VlZBF8Vw93 \n", + "17868521 1537542535.379473 CdWSpn4EA2vXfUBqI7 \n", + "17868522 1537542504.559192 ClKCqX3JOxjrti9lLh \n", + "17868523 1537542573.267978 CDsdf81FPzUzFBBRBe \n", + "17868524 #close 2019-01-03-20-02-05 benign - \n", + "\n", + " uid id.orig_h id.orig_p id.resp_h \\\n", + "0 string addr port addr \n", + "1 0.0.0.0 68 255.255.255.255 67 \n", + "2 255.255.255.255 68 192.168.1.1 67 \n", + "3 fe80::5bcc:698e:39d5:cdf 5353 ff02::fb 5353 \n", + "4 fe80::5bcc:698e:39d5:cdf 5353 ff02::fb 5353 \n", + "... ... ... ... ... \n", + "17868520 fe80::106c:9e5b:3af8:9cf 143 ff02::16 0 \n", + "17868521 fe80::183b:7564:dbcc:3eca 143 ff02::16 0 \n", + "17868522 fe80::1847:a1bd:8d13:f43c 143 ff02::16 0 \n", + "17868523 fe80::1c5c:213d:7cf8:890 143 ff02::16 0 \n", + "17868524 NaN NaN NaN NaN \n", + "\n", + " id.resp_p proto service duration orig_bytes resp_bytes \\\n", + "0 port enum string interval count count \n", + "1 udp dhcp 30.004642 8768 0 S0 \n", + "2 udp dhcp 0.004564 0 3900 SHR \n", + "3 udp dns 3.948539 876 0 S0 \n", + "4 udp dns 3.768179 876 0 S0 \n", + "... ... ... ... ... ... ... \n", + "17868520 icmp - 1.945701 40 0 OTH \n", + "17868521 icmp - 1.024087 40 0 OTH \n", + "17868522 icmp - 59.390351 80 0 OTH \n", + "17868523 icmp - - - - OTH \n", + "17868524 NaN NaN NaN NaN NaN NaN \n", + "\n", + " conn_state local_orig local_resp missed_bytes history orig_pkts \\\n", + "0 string bool bool count string count \n", + "1 - - 0 D 16 9216 \n", + "2 - - 0 ^d 0 0 \n", + "3 - - 0 D 6 1164 \n", + "4 - - 0 D 6 1164 \n", + "... ... ... ... ... ... ... \n", + "17868520 - - 0 - 2 152 \n", + "17868521 - - 0 - 2 152 \n", + "17868522 - - 0 - 4 304 \n", + "17868523 - - 0 - 1 76 \n", + "17868524 NaN NaN NaN NaN NaN NaN \n", + "\n", + " orig_ip_bytes resp_pkts resp_ip_bytes \\\n", + "0 count count count \n", + "1 0 0 - benign - \n", + "2 13 4264 - benign - \n", + "3 0 0 - benign - \n", + "4 0 0 - benign - \n", + "... ... ... ... \n", + "17868520 0 0 - benign - \n", + "17868521 0 0 - benign - \n", + "17868522 0 0 - benign - \n", + "17868523 0 0 - benign - \n", + "17868524 NaN NaN NaN \n", + "\n", + " tunnel_parents label detailed-label label \n", + "0 set[string] string string 0 \n", + "1 NaN 0 \n", + "2 NaN 0 \n", + "3 NaN 0 \n", + "4 NaN 0 \n", + "... ... ... \n", + "17868520 NaN 0 \n", + "17868521 NaN 0 \n", + "17868522 NaN 0 \n", + "17868523 NaN 0 \n", + "17868524 NaN 0 \n", + "\n", + "[17868525 rows x 23 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 5 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qrcgNII2aRBO", + "outputId": "bc70bcc3-512c-4963-a39a-0305e06658a5" + }, + "source": [ + "array = ['#types', '#close']\n", + "noise = df.loc[df['#fields'].isin(array)]\n", + "noise.index" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Int64Index([ 0, 131, 132, 6378426, 6378427, 6378880,\n", + " 6378881, 17833596, 17833597, 17856743, 17856744, 17867148,\n", + " 17867149, 17868524],\n", + " dtype='int64')" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 6 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "dXkrNc5Naoc2" + }, + "source": [ + "df = df.drop(noise.index)" + ], + "execution_count": 7, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "kJnwKnowAFkt" + }, + "source": [ + "DF = df.replace({'-':np.nan})" + ], + "execution_count": 8, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "lKdCOy06q__S" + }, + "source": [ + "DF = DF.fillna(DF.mode().iloc[0])" + ], + "execution_count": 9, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "C_AATawrykZs" + }, + "source": [ + "DF['conn_state'] = DF['conn_state'].replace(np.nan, 0)\n", + "DF['local_orig'] = DF['local_orig'].replace(np.nan, 0)\n", + "DF['tunnel_parents label detailed-label'] = DF['tunnel_parents label detailed-label'].replace(np.nan, 0)" + ], + "execution_count": 10, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 473 + }, + "id": "P3raSwvmCQqZ", + "outputId": "95fad884-f7c5-48fa-98de-7b64d6bc8f79" + }, + "source": [ + "DF" + ], + "execution_count": 11, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#fieldstsuidid.orig_hid.orig_pid.resp_hid.resp_pprotoservicedurationorig_bytesresp_bytesconn_statelocal_origlocal_respmissed_byteshistoryorig_pktsorig_ip_bytesresp_pktsresp_ip_bytestunnel_parents label detailed-labellabel
11562159819.172195CUg3u41D8SwgQw1Job0.0.0.068255.255.255.25567udpdhcp30.00464287680S00.00.00D16921600- benign -0.00
21562159849.173340CUpk9Y381SJuOHyK2d255.255.255.25568192.168.1.167udpdhcp0.00456403900SHR0.00.00^d00134264- benign -0.00
31562159953.959669CC5CQA3Ptzma7a0by4fe80::5bcc:698e:39d5:cdf5353ff02::fb5353udpdns3.9485398760S00.00.00D6116400- benign -0.00
41562159998.302954C3sgcmfvNzjNoY0Kdfe80::5bcc:698e:39d5:cdf5353ff02::fb5353udpdns3.7681798760S00.00.00D6116400- benign -0.00
51562160024.472592CJVaCgoBTEsu0jjDife80::4eef:c0ff:fe27:561e5353ff02::fb5353udpdns0.0001144510S00.00.00D1197900- benign -0.00
........................................................................
178685191537542534.223382CrPQ4n1jVFxbm0mcj9192.168.69.13651443239.255.255.2501900udpdns0.00174600S00.00.00D115300- benign -0.00
178685201537542560.979740Ce0LkH33VlZBF8Vw93fe80::106c:9e5b:3af8:9cf143ff02::160icmpdns1.945701400OTH0.00.00S215200- benign -0.00
178685211537542535.379473CdWSpn4EA2vXfUBqI7fe80::183b:7564:dbcc:3eca143ff02::160icmpdns1.024087400OTH0.00.00S215200- benign -0.00
178685221537542504.559192ClKCqX3JOxjrti9lLhfe80::1847:a1bd:8d13:f43c143ff02::160icmpdns59.390351800OTH0.00.00S430400- benign -0.00
178685231537542573.267978CDsdf81FPzUzFBBRBefe80::1c5c:213d:7cf8:890143ff02::160icmpdns0.00174600OTH0.00.00S17600- benign -0.00
\n", + "

17868511 rows × 23 columns

\n", + "
" + ], + "text/plain": [ + " #fields ts uid \\\n", + "1 1562159819.172195 CUg3u41D8SwgQw1Job 0.0.0.0 \n", + "2 1562159849.173340 CUpk9Y381SJuOHyK2d 255.255.255.255 \n", + "3 1562159953.959669 CC5CQA3Ptzma7a0by4 fe80::5bcc:698e:39d5:cdf \n", + "4 1562159998.302954 C3sgcmfvNzjNoY0Kd fe80::5bcc:698e:39d5:cdf \n", + "5 1562160024.472592 CJVaCgoBTEsu0jjDi fe80::4eef:c0ff:fe27:561e \n", + "... ... ... ... \n", + "17868519 1537542534.223382 CrPQ4n1jVFxbm0mcj9 192.168.69.136 \n", + "17868520 1537542560.979740 Ce0LkH33VlZBF8Vw93 fe80::106c:9e5b:3af8:9cf \n", + "17868521 1537542535.379473 CdWSpn4EA2vXfUBqI7 fe80::183b:7564:dbcc:3eca \n", + "17868522 1537542504.559192 ClKCqX3JOxjrti9lLh fe80::1847:a1bd:8d13:f43c \n", + "17868523 1537542573.267978 CDsdf81FPzUzFBBRBe fe80::1c5c:213d:7cf8:890 \n", + "\n", + " id.orig_h id.orig_p id.resp_h id.resp_p proto service \\\n", + "1 68 255.255.255.255 67 udp dhcp 30.004642 \n", + "2 68 192.168.1.1 67 udp dhcp 0.004564 \n", + "3 5353 ff02::fb 5353 udp dns 3.948539 \n", + "4 5353 ff02::fb 5353 udp dns 3.768179 \n", + "5 5353 ff02::fb 5353 udp dns 0.000114 \n", + "... ... ... ... ... ... ... \n", + "17868519 51443 239.255.255.250 1900 udp dns 0.001746 \n", + "17868520 143 ff02::16 0 icmp dns 1.945701 \n", + "17868521 143 ff02::16 0 icmp dns 1.024087 \n", + "17868522 143 ff02::16 0 icmp dns 59.390351 \n", + "17868523 143 ff02::16 0 icmp dns 0.001746 \n", + "\n", + " duration orig_bytes resp_bytes conn_state local_orig local_resp \\\n", + "1 8768 0 S0 0.0 0.0 0 \n", + "2 0 3900 SHR 0.0 0.0 0 \n", + "3 876 0 S0 0.0 0.0 0 \n", + "4 876 0 S0 0.0 0.0 0 \n", + "5 451 0 S0 0.0 0.0 0 \n", + "... ... ... ... ... ... ... \n", + "17868519 0 0 S0 0.0 0.0 0 \n", + "17868520 40 0 OTH 0.0 0.0 0 \n", + "17868521 40 0 OTH 0.0 0.0 0 \n", + "17868522 80 0 OTH 0.0 0.0 0 \n", + "17868523 0 0 OTH 0.0 0.0 0 \n", + "\n", + " missed_bytes history orig_pkts orig_ip_bytes resp_pkts \\\n", + "1 D 16 9216 0 0 \n", + "2 ^d 0 0 13 4264 \n", + "3 D 6 1164 0 0 \n", + "4 D 6 1164 0 0 \n", + "5 D 11 979 0 0 \n", + "... ... ... ... ... ... \n", + "17868519 D 1 153 0 0 \n", + "17868520 S 2 152 0 0 \n", + "17868521 S 2 152 0 0 \n", + "17868522 S 4 304 0 0 \n", + "17868523 S 1 76 0 0 \n", + "\n", + " resp_ip_bytes tunnel_parents label detailed-label label \n", + "1 - benign - 0.0 0 \n", + "2 - benign - 0.0 0 \n", + "3 - benign - 0.0 0 \n", + "4 - benign - 0.0 0 \n", + "5 - benign - 0.0 0 \n", + "... ... ... ... \n", + "17868519 - benign - 0.0 0 \n", + "17868520 - benign - 0.0 0 \n", + "17868521 - benign - 0.0 0 \n", + "17868522 - benign - 0.0 0 \n", + "17868523 - benign - 0.0 0 \n", + "\n", + "[17868511 rows x 23 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 11 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "rII_hqx8V0tU" + }, + "source": [ + "X = DF.iloc[::,:DF.shape[1]-1]\n", + "\n", + "#Y = DF.iloc[::,-1].values\n", + "Y = DF.iloc[::,-1]" + ], + "execution_count": 12, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MIdqfG85FZ7g", + "outputId": "1565e933-5a1c-4ffe-b65b-f1a5d5542b2a" + }, + "source": [ + "from sklearn.experimental import enable_iterative_imputer \n", + "from sklearn.impute import IterativeImputer\n", + "from sklearn.preprocessing import OrdinalEncoder,LabelEncoder\n", + "from sklearn.ensemble import ExtraTreesRegressor\n", + "encoder = OrdinalEncoder()\n", + "imputer = IterativeImputer(ExtraTreesRegressor())\n", + "le = LabelEncoder()\n", + "# create a list of categorical columns to iterate over\n", + "def encode(data):\n", + " '''function to encode non-null data and replace it in the original data'''\n", + " #retains only non-null values\n", + " nonulls = np.array(data.dropna())\n", + " #reshapes the data for encoding\n", + " impute_reshape = nonulls.reshape(-1,1)\n", + " #encode date\n", + " impute_ordinal = encoder.fit_transform(impute_reshape)\n", + " #Assign back encoded values to non-null values\n", + " data.loc[data.notnull()] = np.squeeze(impute_ordinal)\n", + " return data\n", + "\n", + "#create a for loop to iterate through each column in the data\n", + "for columns in X.columns:\n", + " print(columns)\n", + " encode(X[columns])" + ], + "execution_count": 13, + "outputs": [ + { + "output_type": "stream", + "text": [ + "#fields\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "ts\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "uid\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "id.orig_h\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "id.orig_p\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "id.resp_h\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "id.resp_p\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "proto\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "service\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "duration\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "orig_bytes\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "resp_bytes\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "conn_state\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "local_orig\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "local_resp\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "missed_bytes\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "history\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "orig_pkts\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "orig_ip_bytes\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "resp_pkts\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "resp_ip_bytes\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " iloc._setitem_with_indexer(indexer, value)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "tunnel_parents label detailed-label\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "CHZZ8JZqbCwy" + }, + "source": [ + "import matplotlib.pyplot as plt\n", + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.decomposition import PCA\n", + "\n", + "# Create a scaler object\n", + "sc = StandardScaler()\n", + "\n", + "# Fit the scaler to the features and transform\n", + "Xtrain_std = sc.fit_transform(X)" + ], + "execution_count": 14, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Fi1KY4QK7Gu5" + }, + "source": [ + "from sklearn import decomposition\n", + "# Create a pca object with the k components as a parameter\n", + "pca = decomposition.PCA(n_components=8)\n", + "\n", + "# Fit the PCA and transform the data\n", + "Xa = pca.fit_transform(Xtrain_std)" + ], + "execution_count": 15, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "kzag57Nm7Grw", + "outputId": "c97ebf2b-2fc2-4a0a-89d8-7c1a664ccaae" + }, + "source": [ + "# Oversample and plot imbalanced dataset with ADASYN\n", + "from collections import Counter\n", + "from sklearn.datasets import make_classification\n", + "from imblearn.over_sampling import ADASYN\n", + "from matplotlib import pyplot\n", + "from numpy import where\n", + "# counter = Counter(Y)\n", + "# print(counter)\n", + "# transform the dataset\n", + "oversample = ADASYN()\n", + "x, y = oversample.fit_resample(Xa, Y)\n", + "# summarize the new class distribution\n", + "counter = Counter(y)\n", + "print(counter)" + ], + "execution_count": 16, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/sklearn/externals/six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n", + " \"(https://pypi.org/project/six/).\", FutureWarning)\n", + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n", + " warnings.warn(message, FutureWarning)\n", + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n", + " warnings.warn(msg, category=FutureWarning)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Counter({0: 17866574, 1: 17866555})\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "clB1M48vbCzk" + }, + "source": [ + "from sklearn.model_selection import train_test_split\n", + "X_train, X_test, Y_train, Y_test = train_test_split(x, y, test_size=0.2)\n", + "#X_val, X_test, Y_val, Y_test = train_test_split(X_val_and_test, Y_val_and_test, test_size=0.2)" + ], + "execution_count": 29, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "S32ki_Z2bC13" + }, + "source": [ + "import keras\n", + "from tensorflow.keras.models import Sequential\n", + "from tensorflow.keras.layers import Dense\n", + "from keras.layers import Dropout\n", + "from keras import regularizers\n", + "from keras.constraints import maxnorm\n", + "from keras.optimizers import SGD" + ], + "execution_count": 30, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "85z1VzjJ9weZ" + }, + "source": [ + "model = Sequential([\n", + " Dense(120, activation='relu',kernel_regularizer=regularizers.l2(0.01), input_shape=(8,)),\n", + " Dropout(0.3),\n", + " Dense(65, activation='relu',kernel_regularizer=regularizers.l2(0.01)),\n", + " Dropout(0.3),\n", + " Dense(35, activation='relu',kernel_regularizer=regularizers.l2(0.01)),\n", + " Dropout(0.3),\n", + " Dense(16, activation='relu',kernel_regularizer=regularizers.l2(0.01)),\n", + " Dropout(0.3),\n", + " Dense(1, activation='sigmoid'),\n", + "])" + ], + "execution_count": 31, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "NAhqlOLK9whO" + }, + "source": [ + "epochs = 1\n", + "learning_rate = 0.01\n", + "momentum = 0.9\n", + "sgd = SGD(learning_rate=learning_rate, momentum=momentum)\n", + "model.compile(optimizer=sgd,\n", + " loss='binary_crossentropy',\n", + " metrics=['accuracy'])" + ], + "execution_count": 32, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 409 + }, + "id": "iNHO_JJo9wjy", + "outputId": "fae22d48-c2a4-4979-be3d-a6aa3137bce3" + }, + "source": [ + "import time\n", + "from datetime import timedelta\n", + "start_time = time.monotonic()\n", + "\n", + "\n", + "\n", + "\n", + "hist = model.fit(X_train, Y_train,\n", + " batch_size=512, epochs=epochs,)\n", + " # validation_data=(X_val, Y_val))\n", + "\n", + "\n", + "\n", + "\n", + "end_time = time.monotonic()\n", + "print(timedelta(seconds=end_time - start_time))" + ], + "execution_count": 33, + "outputs": [ + { + "output_type": "stream", + "text": [ + " 2510/55834 [>.............................] - ETA: 4:06 - loss: 0.2218 - accuracy: 0.9927" + ], + "name": "stdout" + }, + { + "output_type": "error", + "ename": "KeyboardInterrupt", + "evalue": "ignored", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m hist = model.fit(X_train, Y_train,\n\u001b[0;32m----> 9\u001b[0;31m batch_size=512, epochs=epochs,)\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;31m# validation_data=(X_val, Y_val))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1181\u001b[0m _r=1):\n\u001b[1;32m 1182\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1183\u001b[0;31m \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1184\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1185\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 887\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 888\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 890\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 891\u001b[0m \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 915\u001b[0m \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 916\u001b[0m \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 918\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 919\u001b[0m \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 3022\u001b[0m filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m 3023\u001b[0m return graph_function._call_flat(\n\u001b[0;32m-> 3024\u001b[0;31m filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m 3025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3026\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m 1959\u001b[0m \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1960\u001b[0m return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1961\u001b[0;31m ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m 1962\u001b[0m forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m 1963\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m 594\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 595\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 596\u001b[0;31m ctx=ctx)\n\u001b[0m\u001b[1;32m 597\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 598\u001b[0m outputs = execute.execute_with_cancellation(\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 59\u001b[0m tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m 61\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Nq11aU9__pbZ", + "outputId": "78a1c178-0bdb-41d9-e6e6-c43c81066ce7" + }, + "source": [ + "len(X_train[0])" + ], + "execution_count": 24, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "8" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 24 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "SiR9I5_W9wmM" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Zzy79yYM9wox" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "MGQ5uWso9wq6" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jC8jjViQPVzq" + }, + "source": [ + "## features = 22" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "2kjVRPPyZrQ9" + }, + "source": [ + "X = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/IoT-23/data_input/X_23.csv',error_bad_lines=False,low_memory=False).values\n", + "Y = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/IoT-23/data_input/Y_23.csv',error_bad_lines=False,low_memory=False).values" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qcf3CFVGLhYl", + "outputId": "cbae1de2-528f-4a37-ef15-1f8502290687" + }, + "source": [ + "# Oversample and plot imbalanced dataset with ADASYN\n", + "from collections import Counter\n", + "from sklearn.datasets import make_classification\n", + "from imblearn.over_sampling import ADASYN\n", + "from matplotlib import pyplot\n", + "from numpy import where\n", + "# counter = Counter(Y)\n", + "# print(counter)\n", + "# transform the dataset\n", + "oversample = ADASYN()\n", + "x, y = oversample.fit_resample(X, Y)\n", + "# summarize the new class distribution\n", + "counter = Counter(y)\n", + "print(counter)\n" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/sklearn/externals/six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n", + " \"(https://pypi.org/project/six/).\", FutureWarning)\n", + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n", + " warnings.warn(message, FutureWarning)\n", + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n", + " y = column_or_1d(y, warn=True)\n", + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n", + " warnings.warn(msg, category=FutureWarning)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Counter({1: 17866555, 0: 17866341})\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "aAeU2cDBkEeq" + }, + "source": [ + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.decomposition import PCA\n", + "\n", + "# Create a scaler object\n", + "sc = StandardScaler()\n", + "\n", + "# Fit the scaler to the features and transform\n", + "Xa = sc.fit_transform(x)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "8J4cvKtykQrR" + }, + "source": [ + "from sklearn.model_selection import train_test_split\n", + "X_train, X_val_and_test, Y_train, Y_val_and_test = train_test_split(Xa, y, test_size=0.2)\n", + "X_val, X_test, Y_val, Y_test = train_test_split(X_val_and_test, Y_val_and_test, test_size=0.2)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "og1JkQxFNrIA", + "outputId": "29e6a9bd-1862-434c-9d85-c3d36a652790" + }, + "source": [ + "print(\"X_train\",len(X_train))\n", + "print(\"Y_train\",len(Y_train))\n", + "print(\"X_test\",len(X_test))\n", + "print(\"Y_test\",len(Y_test))\n", + "print(\"X_val\",len(X_val))\n", + "print(\"Y_val\",len(Y_val))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "X_train 28586316\n", + "Y_train 28586316\n", + "X_test 1429316\n", + "Y_test 1429316\n", + "X_val 5717264\n", + "Y_val 5717264\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "_553VN17k8Qg" + }, + "source": [ + "import keras\n", + "from tensorflow.keras.models import Sequential\n", + "from tensorflow.keras.layers import Dense\n", + "from keras.layers import Dropout\n", + "from keras import regularizers\n", + "from keras.constraints import maxnorm\n", + "from keras.optimizers import SGD" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "WWjvplhCk8t8" + }, + "source": [ + "model = Sequential([\n", + " Dense(120, activation='relu',kernel_regularizer=regularizers.l2(0.01), input_shape=(22,)),\n", + " Dropout(0.3),\n", + " Dense(65, activation='relu',kernel_regularizer=regularizers.l2(0.01)),\n", + " Dropout(0.3),\n", + " Dense(35, activation='relu',kernel_regularizer=regularizers.l2(0.01)),\n", + " Dropout(0.3),\n", + " Dense(16, activation='relu',kernel_regularizer=regularizers.l2(0.01)),\n", + " Dropout(0.3),\n", + " Dense(1, activation='sigmoid'),\n", + "])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3ahScerJlVcs", + "outputId": "7c1a58dd-270f-41c1-a330-3a22c4b98c94" + }, + "source": [ + "model.summary()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model: \"sequential_2\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "dense_10 (Dense) (None, 120) 2760 \n", + "_________________________________________________________________\n", + "module_wrapper_8 (ModuleWrap (None, 120) 0 \n", + "_________________________________________________________________\n", + "dense_11 (Dense) (None, 65) 7865 \n", + "_________________________________________________________________\n", + "module_wrapper_9 (ModuleWrap (None, 65) 0 \n", + "_________________________________________________________________\n", + "dense_12 (Dense) (None, 35) 2310 \n", + "_________________________________________________________________\n", + "module_wrapper_10 (ModuleWra (None, 35) 0 \n", + "_________________________________________________________________\n", + "dense_13 (Dense) (None, 16) 576 \n", + "_________________________________________________________________\n", + "module_wrapper_11 (ModuleWra (None, 16) 0 \n", + "_________________________________________________________________\n", + "dense_14 (Dense) (None, 1) 17 \n", + "=================================================================\n", + "Total params: 13,528\n", + "Trainable params: 13,528\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "MM5MmmC_lXDP" + }, + "source": [ + "\n", + "epochs = 4\n", + "learning_rate = 0.01\n", + "momentum = 0.9\n", + "sgd = SGD(learning_rate=learning_rate, momentum=momentum)\n", + "model.compile(optimizer=sgd,\n", + " loss='binary_crossentropy',\n", + " metrics=['accuracy'])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "BS1KejyyqFUk", + "outputId": "0620af30-da5d-4847-917c-ea3ac39632a6" + }, + "source": [ + "import time\n", + "from datetime import timedelta\n", + "start_time = time.monotonic()\n", + "\n", + "\n", + "\n", + "\n", + "hist = model.fit(X_train, Y_train,\n", + " batch_size=512, epochs=epochs,\n", + " validation_data=(X_val, Y_val))\n", + "\n", + "\n", + "\n", + "\n", + "end_time = time.monotonic()\n", + "print(timedelta(seconds=end_time - start_time))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 1/4\n", + "55833/55833 [==============================] - 382s 7ms/step - loss: 0.0555 - accuracy: 0.9979 - val_loss: 0.0327 - val_accuracy: 0.9993\n", + "Epoch 2/4\n", + "55833/55833 [==============================] - 384s 7ms/step - loss: 0.0365 - accuracy: 0.9987 - val_loss: 0.0300 - val_accuracy: 0.9993\n", + "Epoch 3/4\n", + "55833/55833 [==============================] - 388s 7ms/step - loss: 0.0336 - accuracy: 0.9988 - val_loss: 0.0278 - val_accuracy: 0.9994\n", + "Epoch 4/4\n", + "55833/55833 [==============================] - 391s 7ms/step - loss: 0.0318 - accuracy: 0.9989 - val_loss: 0.0262 - val_accuracy: 0.9998\n", + "0:25:49.625131\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "jmbM1ThYL642", + "outputId": "45b902e3-bfdb-4656-d098-0cb614b04451" + }, + "source": [ + "model.save(\"/content/drive/MyDrive/Colab Notebooks/IoT-23/model_22features\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "WARNING:absl:Found untraced functions such as dropout_8_layer_call_and_return_conditional_losses, dropout_8_layer_call_fn, dropout_9_layer_call_and_return_conditional_losses, dropout_9_layer_call_fn, dropout_10_layer_call_and_return_conditional_losses while saving (showing 5 of 20). These functions will not be directly callable after loading.\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /content/drive/MyDrive/Colab Notebooks/IoT-23/model_22features/assets\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /content/drive/MyDrive/Colab Notebooks/IoT-23/model_22features/assets\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aHPsOZVJqO5L", + "outputId": "de110015-1bee-43a5-9986-6107de51d5f1" + }, + "source": [ + "import time\n", + "from datetime import timedelta\n", + "start_time = time.monotonic()\n", + "\n", + "yout = model.predict_classes(X_test)\n", + "\n", + "end_time = time.monotonic()\n", + "print(timedelta(seconds=end_time - start_time))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/sequential.py:455: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`, if your model does multi-class classification (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`, if your model does binary classification (e.g. if it uses a `sigmoid` last-layer activation).\n", + " warnings.warn('`model.predict_classes()` is deprecated and '\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "0:00:40.263582\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "I0qVTfm_LYj7" + }, + "source": [ + "result = yout.ravel()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "mXl_lXGLLXuJ", + "outputId": "f36aae73-5546-466f-bc23-c083981f2f4f" + }, + "source": [ + "def my_confusion_matrix(y_true, y_pred):\n", + " N = np.unique(y_true).shape[0] # number of classes \n", + " cm = np.zeros((N, N))\n", + " for n in range(y_true.shape[0]):\n", + " cm[y_true[n], y_pred[n]] += 1\n", + " return cm \n", + "\n", + "cnf_matrix = my_confusion_matrix(Y_test, result)\n", + "print('Confusion matrix:')\n", + "print(cnf_matrix)\n", + "print('\\nAccuracy:', np.diagonal(cnf_matrix).sum()/cnf_matrix.sum())\n" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Confusion matrix:\n", + "[[7.14238e+05 0.00000e+00]\n", + " [2.86000e+02 7.14792e+05]]\n", + "\n", + "Accuracy: 0.9997999042898841\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f7Dp9Fj9Lmfp" + }, + "source": [ + "##TH features = 16\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 647 + }, + "id": "Is6EE1njLt7y", + "outputId": "e0a28059-1c7d-439f-e35a-0a568111d553" + }, + "source": [ + "X = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/IoT-23/data_input/X_23.csv',error_bad_lines=False,low_memory=False)\n", + "Y = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/IoT-23/data_input/Y_23.csv',error_bad_lines=False,low_memory=False).values" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "error", + "ename": "KeyboardInterrupt", + "evalue": "ignored", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/Colab Notebooks/IoT-23/data_input/X_23.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0merror_bad_lines\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlow_memory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mY\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/Colab Notebooks/IoT-23/data_input/Y_23.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0merror_bad_lines\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlow_memory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m 686\u001b[0m )\n\u001b[1;32m 687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 688\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 689\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 458\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 459\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 460\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 461\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 462\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 1196\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1197\u001b[0m \u001b[0mnrows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_validate_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"nrows\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1198\u001b[0;31m \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1200\u001b[0m \u001b[0;31m# May alter columns / col_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 2155\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2156\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2157\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2158\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2159\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_first_chunk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.read\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_column_data\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_tokens\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_with_dtype\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/dtypes/common.py\u001b[0m in \u001b[0;36mis_categorical_dtype\u001b[0;34m(arr_or_dtype)\u001b[0m\n\u001b[1;32m 528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 529\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mis_categorical_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr_or_dtype\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 531\u001b[0m \"\"\"\n\u001b[1;32m 532\u001b[0m \u001b[0mCheck\u001b[0m \u001b[0mwhether\u001b[0m \u001b[0man\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlike\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mCategorical\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "qb-_wKcLLzdH" + }, + "source": [ + "cut = ['local_orig','conn_state','tunnel_parents label detailed-label',\n", + " 'proto','orig_bytes','duration','service']\n", + "X = X.drop(cut, axis=1)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 439 + }, + "id": "Br3sr1VGeTyQ", + "outputId": "a1b568ee-3302-4324-bba0-6bc932ef931f" + }, + "source": [ + "X" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#fieldstsuidid.orig_hid.orig_pid.resp_hid.resp_presp_byteslocal_respmissed_byteshistoryorig_pktsorig_ip_bytesresp_pktsresp_ip_bytes
017868380.08842789.01.061981.010823557.04008.02.06.00.05.019.0852.00.00.010.0
117868381.08887696.05309.061981.08003385.04008.02.012.00.0112.00.00.012.0271.010.0
217868387.03485594.08749.048369.016581289.03112.02.06.00.05.090.059.00.00.010.0
317868388.01119675.08749.048369.016581289.03112.02.06.00.05.090.059.00.00.010.0
417868389.05625874.08748.048369.016581289.03112.02.06.00.05.04.0873.00.00.010.0
................................................
1786850617844765.015392839.03196.046051.010657502.0685.02.06.00.05.01.0196.00.00.010.0
1786850717844778.011530428.08740.04780.016581282.00.00.00.00.023.031.0195.00.00.010.0
1786850817844767.011391762.08742.04780.016581282.00.00.00.00.023.031.0195.00.00.010.0
1786850917844754.013639170.08743.04780.016581282.00.00.00.00.023.065.0402.00.00.010.0
1786851017844782.04003582.08745.04780.016581282.00.00.00.00.023.01.0775.00.00.010.0
\n", + "

17868511 rows × 15 columns

\n", + "
" + ], + "text/plain": [ + " #fields ts ... resp_pkts resp_ip_bytes\n", + "0 17868380.0 8842789.0 ... 0.0 10.0\n", + "1 17868381.0 8887696.0 ... 271.0 10.0\n", + "2 17868387.0 3485594.0 ... 0.0 10.0\n", + "3 17868388.0 1119675.0 ... 0.0 10.0\n", + "4 17868389.0 5625874.0 ... 0.0 10.0\n", + "... ... ... ... ... ...\n", + "17868506 17844765.0 15392839.0 ... 0.0 10.0\n", + "17868507 17844778.0 11530428.0 ... 0.0 10.0\n", + "17868508 17844767.0 11391762.0 ... 0.0 10.0\n", + "17868509 17844754.0 13639170.0 ... 0.0 10.0\n", + "17868510 17844782.0 4003582.0 ... 0.0 10.0\n", + "\n", + "[17868511 rows x 15 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 12 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "f-9sKGjWNMUi", + "outputId": "7c332bfa-594b-4123-9be7-a4c0560e53d4" + }, + "source": [ + "# Oversample and plot imbalanced dataset with ADASYN\n", + "from collections import Counter\n", + "from sklearn.datasets import make_classification\n", + "from imblearn.over_sampling import ADASYN\n", + "from matplotlib import pyplot\n", + "from numpy import where\n", + "# counter = Counter(Y)\n", + "# print(counter)\n", + "# transform the dataset\n", + "oversample = ADASYN()\n", + "x, y = oversample.fit_resample(X, Y)\n", + "# summarize the new class distribution\n", + "counter = Counter(y)\n", + "print(counter)\n" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/sklearn/externals/six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n", + " \"(https://pypi.org/project/six/).\", FutureWarning)\n", + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n", + " warnings.warn(message, FutureWarning)\n", + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n", + " y = column_or_1d(y, warn=True)\n", + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n", + " warnings.warn(msg, category=FutureWarning)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Counter({1: 17866555, 0: 17866508})\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "-vsVfYyifEoM" + }, + "source": [ + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.decomposition import PCA\n", + "\n", + "# Create a scaler object\n", + "sc = StandardScaler()\n", + "\n", + "# Fit the scaler to the features and transform\n", + "Xa = sc.fit_transform(x)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "sEtvlmDefIEy" + }, + "source": [ + "from sklearn.model_selection import train_test_split\n", + "X_train, X_val_and_test, Y_train, Y_val_and_test = train_test_split(Xa, y, test_size=0.2)\n", + "X_val, X_test, Y_val, Y_test = train_test_split(X_val_and_test, Y_val_and_test, test_size=0.2)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "CgudSK1ZfKth", + "outputId": "ceee659f-fb21-436c-e12b-c93f2290b7f8" + }, + "source": [ + "print(\"X_train\",len(X_train))\n", + "print(\"Y_train\",len(Y_train))\n", + "print(\"X_test\",len(X_test))\n", + "print(\"Y_test\",len(Y_test))\n", + "print(\"X_val\",len(X_val))\n", + "print(\"Y_val\",len(Y_val))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "X_train 28586450\n", + "Y_train 28586450\n", + "X_test 1429323\n", + "Y_test 1429323\n", + "X_val 5717290\n", + "Y_val 5717290\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GGtXYByVfNg0" + }, + "source": [ + "import keras\n", + "from tensorflow.keras.models import Sequential\n", + "from tensorflow.keras.layers import Dense\n", + "from keras.layers import Dropout\n", + "from keras import regularizers\n", + "from keras.constraints import maxnorm\n", + "from keras.optimizers import SGD" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "TLV5feAhfP7M" + }, + "source": [ + "model1 = Sequential([\n", + " Dense(120, activation='relu',kernel_regularizer=regularizers.l2(0.01), input_shape=(15,)),\n", + " Dropout(0.3),\n", + " Dense(65, activation='relu',kernel_regularizer=regularizers.l2(0.01)),\n", + " Dropout(0.3),\n", + " Dense(35, activation='relu',kernel_regularizer=regularizers.l2(0.01)),\n", + " Dropout(0.3),\n", + " Dense(16, activation='relu',kernel_regularizer=regularizers.l2(0.01)),\n", + " Dropout(0.3),\n", + " Dense(1, activation='sigmoid'),\n", + "])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Xo1QukYZfT-U", + "outputId": "70217104-e3a4-4029-c4b3-9521bd2c612b" + }, + "source": [ + "model1.summary()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model: \"sequential_1\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "dense_5 (Dense) (None, 120) 1920 \n", + "_________________________________________________________________\n", + "module_wrapper_4 (ModuleWrap (None, 120) 0 \n", + "_________________________________________________________________\n", + "dense_6 (Dense) (None, 65) 7865 \n", + "_________________________________________________________________\n", + "module_wrapper_5 (ModuleWrap (None, 65) 0 \n", + "_________________________________________________________________\n", + "dense_7 (Dense) (None, 35) 2310 \n", + "_________________________________________________________________\n", + "module_wrapper_6 (ModuleWrap (None, 35) 0 \n", + "_________________________________________________________________\n", + "dense_8 (Dense) (None, 16) 576 \n", + "_________________________________________________________________\n", + "module_wrapper_7 (ModuleWrap (None, 16) 0 \n", + "_________________________________________________________________\n", + "dense_9 (Dense) (None, 1) 17 \n", + "=================================================================\n", + "Total params: 12,688\n", + "Trainable params: 12,688\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "dpZL56QMfV83" + }, + "source": [ + "epochs = 4\n", + "learning_rate = 0.01\n", + "momentum = 0.9\n", + "sgd = SGD(learning_rate=learning_rate, momentum=momentum)\n", + "model1.compile(optimizer=sgd,\n", + " loss='binary_crossentropy',\n", + " metrics=['accuracy'])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "8R6DgRc6fY_p", + "outputId": "e3cd4502-8165-431c-8502-1df15dee17d7" + }, + "source": [ + "import time\n", + "from datetime import timedelta\n", + "start_time = time.monotonic()\n", + "\n", + "\n", + "\n", + "\n", + "hist = model1.fit(X_train, Y_train,\n", + " batch_size=512, epochs=epochs,\n", + " validation_data=(X_val, Y_val))\n", + "\n", + "\n", + "\n", + "\n", + "end_time = time.monotonic()\n", + "print(timedelta(seconds=end_time - start_time))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 1/4\n", + "55833/55833 [==============================] - 316s 6ms/step - loss: 0.0561 - accuracy: 0.9983 - val_loss: 0.0337 - val_accuracy: 0.9994\n", + "Epoch 2/4\n", + "55833/55833 [==============================] - 313s 6ms/step - loss: 0.0366 - accuracy: 0.9991 - val_loss: 0.0302 - val_accuracy: 0.9994\n", + "Epoch 3/4\n", + "55833/55833 [==============================] - 316s 6ms/step - loss: 0.0336 - accuracy: 0.9992 - val_loss: 0.0279 - val_accuracy: 0.9995\n", + "Epoch 4/4\n", + "55833/55833 [==============================] - 315s 6ms/step - loss: 0.0317 - accuracy: 0.9993 - val_loss: 0.0267 - val_accuracy: 0.9996\n", + "0:21:02.058668\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "gHqRwmtVgiiZ", + "outputId": "0ad22b16-c2e5-41cf-9c73-4d89fbde1ecc" + }, + "source": [ + "import time\n", + "from datetime import timedelta\n", + "start_time = time.monotonic()\n", + "\n", + "yout = model1.predict_classes(X_test)\n", + "\n", + "end_time = time.monotonic()\n", + "print(timedelta(seconds=end_time - start_time))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/sequential.py:455: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`, if your model does multi-class classification (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`, if your model does binary classification (e.g. if it uses a `sigmoid` last-layer activation).\n", + " warnings.warn('`model.predict_classes()` is deprecated and '\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "0:00:41.139643\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "6W9uSV0AgyFf" + }, + "source": [ + "result = yout.ravel()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "zK82Gjj_g15Q", + "outputId": "5e91431f-3dc0-405e-f0fa-3e851372ba3c" + }, + "source": [ + "def my_confusion_matrix(y_true, y_pred):\n", + " N = np.unique(y_true).shape[0] # number of classes \n", + " cm = np.zeros((N, N))\n", + " for n in range(y_true.shape[0]):\n", + " cm[y_true[n], y_pred[n]] += 1\n", + " return cm \n", + "\n", + "cnf_matrix = my_confusion_matrix(Y_test, result)\n", + "print('Confusion matrix:')\n", + "print(cnf_matrix)\n", + "print('\\nAccuracy:', np.diagonal(cnf_matrix).sum()/cnf_matrix.sum())" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Confusion matrix:\n", + "[[7.14528e+05 0.00000e+00]\n", + " [6.03000e+02 7.14192e+05]]\n", + "\n", + "Accuracy: 0.999578121950042\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "BmH5zGgwlp5u" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ne1uUAs5lvw1" + }, + "source": [ + "## features = 8" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "L56oeQkol5Z9" + }, + "source": [ + "X = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/IoT-23/data_input/X_23.csv',error_bad_lines=False,low_memory=False).values\n", + "Y = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/IoT-23/data_input/Y_23.csv',error_bad_lines=False,low_memory=False).values" + ], + "execution_count": 3, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "5hXu48Otl5Z9" + }, + "source": [ + "# cut = ['local_orig','conn_state','tunnel_parents label detailed-label',\n", + "# 'proto','orig_bytes','duration','service']\n", + "# X = X.drop(cut, axis=1)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "tNhhuR5pl5Z_" + }, + "source": [ + "import matplotlib.pyplot as plt\n", + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.decomposition import PCA\n", + "\n", + "# Create a scaler object\n", + "sc = StandardScaler()\n", + "\n", + "# Fit the scaler to the features and transform\n", + "Xtrain_std = sc.fit_transform(X)" + ], + "execution_count": 4, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "yAcnIm1jmIaj" + }, + "source": [ + "from sklearn import decomposition\n", + "# Create a pca object with the k components as a parameter\n", + "pca = decomposition.PCA(n_components=2)\n", + "\n", + "# Fit the PCA and transform the data\n", + "Xa = pca.fit_transform(Xtrain_std)" + ], + "execution_count": 14, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "AfRNX01psjJE", + "outputId": "3e1bcf87-ef7c-4357-c836-049bbb140a73" + }, + "source": [ + "# Oversample and plot imbalanced dataset with ADASYN\n", + "from collections import Counter\n", + "from sklearn.datasets import make_classification\n", + "from imblearn.over_sampling import ADASYN\n", + "from matplotlib import pyplot\n", + "from numpy import where\n", + "# counter = Counter(Y)\n", + "# print(counter)\n", + "# transform the dataset\n", + "oversample = ADASYN()\n", + "x, y = oversample.fit_resample(Xa, Y)\n", + "# summarize the new class distribution\n", + "counter = Counter(y)\n", + "print(counter)" + ], + "execution_count": 15, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n", + " y = column_or_1d(y, warn=True)\n", + "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n", + " warnings.warn(msg, category=FutureWarning)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Counter({1: 17866555, 0: 17866532})\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hDKPsccbl5Z_" + }, + "source": [ + "from sklearn.model_selection import train_test_split\n", + "X_train, X_val_and_test, Y_train, Y_val_and_test = train_test_split(x, y, test_size=0.2)\n", + "X_val, X_test, Y_val, Y_test = train_test_split(X_val_and_test, Y_val_and_test, test_size=0.2)" + ], + "execution_count": 16, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "sI2z1bLOl5Z_", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "73ac3a7b-f1a9-403b-8a79-9b7574b53c5c" + }, + "source": [ + "print(\"X_train\",len(X_train))\n", + "print(\"Y_train\",len(Y_train))\n", + "print(\"X_test\",len(X_test))\n", + "print(\"Y_test\",len(Y_test))\n", + "print(\"X_val\",len(X_val))\n", + "print(\"Y_val\",len(Y_val))" + ], + "execution_count": 20, + "outputs": [ + { + "output_type": "stream", + "text": [ + "X_train 28586469\n", + "Y_train 28586469\n", + "X_test 1429324\n", + "Y_test 1429324\n", + "X_val 5717294\n", + "Y_val 5717294\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "CDxIanjsl5aA" + }, + "source": [ + "import keras\n", + "from tensorflow.keras.models import Sequential\n", + "from tensorflow.keras.layers import Dense\n", + "from keras.layers import Dropout\n", + "from keras import regularizers\n", + "from keras.constraints import maxnorm\n", + "from keras.optimizers import SGD" + ], + "execution_count": 21, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "IkeUC7fMwK28" + }, + "source": [ + "model3 = Sequential([\n", + " Dense(120, activation='relu', input_shape=(2,)),\n", + " Dropout(0.3),\n", + " Dense(65, activation='relu'),\n", + " Dropout(0.3),\n", + " Dense(35, activation='relu'),\n", + " Dropout(0.3),\n", + " Dense(16, activation='relu'),\n", + " Dropout(0.3),\n", + " Dense(1, activation='sigmoid'),\n", + "])" + ], + "execution_count": 22, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "bMhgwgjKl5aA", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "91600331-805a-4728-b214-5c37a36381e6" + }, + "source": [ + "model3.summary()" + ], + "execution_count": 23, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model: \"sequential_2\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "dense_10 (Dense) (None, 120) 360 \n", + "_________________________________________________________________\n", + "module_wrapper_8 (ModuleWrap (None, 120) 0 \n", + "_________________________________________________________________\n", + "dense_11 (Dense) (None, 65) 7865 \n", + "_________________________________________________________________\n", + "module_wrapper_9 (ModuleWrap (None, 65) 0 \n", + "_________________________________________________________________\n", + "dense_12 (Dense) (None, 35) 2310 \n", + "_________________________________________________________________\n", + "module_wrapper_10 (ModuleWra (None, 35) 0 \n", + "_________________________________________________________________\n", + "dense_13 (Dense) (None, 16) 576 \n", + "_________________________________________________________________\n", + "module_wrapper_11 (ModuleWra (None, 16) 0 \n", + "_________________________________________________________________\n", + "dense_14 (Dense) (None, 1) 17 \n", + "=================================================================\n", + "Total params: 11,128\n", + "Trainable params: 11,128\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "htcWY3Uwl5aB" + }, + "source": [ + "epochs = 1\n", + "learning_rate = 0.01\n", + "momentum = 0.9\n", + "sgd = SGD(learning_rate=learning_rate, momentum=momentum)\n", + "model3.compile(optimizer=sgd,\n", + " loss='binary_crossentropy',\n", + " metrics=['accuracy'])" + ], + "execution_count": 24, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Ju5l3NJMl5aB", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "2febb5b4-23f1-46ac-d166-6d69c1aaf05d" + }, + "source": [ + "import time\n", + "from datetime import timedelta\n", + "start_time = time.monotonic()\n", + "\n", + "\n", + "\n", + "\n", + "hist = model3.fit(X_train, Y_train,\n", + " batch_size=512, epochs=epochs,\n", + " validation_data=(X_val, Y_val))\n", + "\n", + "\n", + "\n", + "\n", + "end_time = time.monotonic()\n", + "print(timedelta(seconds=end_time - start_time))" + ], + "execution_count": 25, + "outputs": [ + { + "output_type": "stream", + "text": [ + "55833/55833 [==============================] - 255s 5ms/step - loss: 0.0220 - accuracy: 0.9957 - val_loss: 0.0183 - val_accuracy: 0.9961\n", + "0:04:15.610815\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "W2A5VnpXl5aB", + "outputId": "359b5d9d-9e66-49f8-d483-136fb60d2758" + }, + "source": [ + "import time\n", + "from datetime import timedelta\n", + "start_time = time.monotonic()\n", + "\n", + "yout = model3.predict_classes(X_test)\n", + "\n", + "end_time = time.monotonic()\n", + "print(timedelta(seconds=end_time - start_time))" + ], + "execution_count": 27, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/sequential.py:455: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`, if your model does multi-class classification (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`, if your model does binary classification (e.g. if it uses a `sigmoid` last-layer activation).\n", + " warnings.warn('`model.predict_classes()` is deprecated and '\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "0:00:27.928609\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GXR-R1cKl5aB" + }, + "source": [ + "result = yout.ravel()" + ], + "execution_count": 28, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Utzcf3T_l5aB", + "outputId": "8b59e9b7-9287-4f12-ce57-d5d4e751b00e" + }, + "source": [ + "def my_confusion_matrix(y_true, y_pred):\n", + " N = np.unique(y_true).shape[0] # number of classes \n", + " cm = np.zeros((N, N))\n", + " for n in range(y_true.shape[0]):\n", + " cm[y_true[n], y_pred[n]] += 1\n", + " return cm \n", + "\n", + "cnf_matrix = my_confusion_matrix(Y_test, result)\n", + "print('Confusion matrix:')\n", + "print(cnf_matrix)\n", + "print('\\nAccuracy:', np.diagonal(cnf_matrix).sum()/cnf_matrix.sum())" + ], + "execution_count": 29, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Confusion matrix:\n", + "[[714410. 0.]\n", + " [ 5540. 709374.]]\n", + "\n", + "Accuracy: 0.9961240418547509\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "bmq6AzrLl5aC" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "L_LX700pVmPE" + }, + "source": [ + "import pandas as pd\n", + "d = {'22 features':[1529775,41139],\n", + " '15 features':[1261235,30673],\n", + " '8 features' : [609007,17201]}\n", + "df = pd.DataFrame(data=d)" + ], + "execution_count": 22, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "PoU1_n4FbM79" + }, + "source": [ + "training_time = df.iloc[0]\n", + "testing_time = df.iloc[1]" + ], + "execution_count": 29, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "wf-IN6sIb85T", + "outputId": "a3eb2db3-d407-4302-86dc-388cfe473729" + }, + "source": [ + "training_time" + ], + "execution_count": 30, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "22 features 1529775\n", + "15 features 1261235\n", + "8 features 609007\n", + "Name: 0, dtype: int64" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 30 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 351 + }, + "id": "dHVrcFjeX5Wy", + "outputId": "f9a63da8-2f51-40de-bbd3-abf257156cdf" + }, + "source": [ + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "\n", + "# a simple line plot\n", + "df.plot(kind='bar',x='22 features',y='15 features')" + ], + "execution_count": 7, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 7 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAE9CAYAAAABCHl5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de1RTZ7438G8SQKigAnKJ0r6OiohWRyq1vh1pexTFoyA6VRlBbb1gD61yxCutClIdO4y3VvA+ra2lY311jVKQAcZLVZzaWnUqR7yP6ACRO3IpFEj2+cPVTHlBCZiww5PvZy3XItnPfvILSb5sf3myo5AkSQIREQlHKXcBRERkGgx4IiJBMeCJiATFgCciEhQDnohIUAx4IiJBmVXAx8fHY/To0fDy8sLNmzcN2qeiogJLlixBQEAAJk6ciMTERBNXSUTUOVjJXcAvjRkzBrNnz0ZYWJjB+0RHR2PkyJHYsmULAKC4uNhU5RERdSpmFfC+vr4tXv/DDz9g06ZNqKmpAQBERkbitddeQ25uLm7evImdO3fqx7q4uHRIrURE5s6sAr4llZWViI2NxZ49e+Dq6oqioiJMnToVqampuH37Ntzc3LBq1Spcu3YNPXv2xIoVK+Dp6Sl32UREsjP7gL98+TLy8vIQHh6uv06hUODevXvQ6XT44YcfsHTpUvj6+iIzMxMRERE4fvy4jBUTEZkHsw94SZLg5eWFL774osXtarVa39oZN24cli9fjrKyMjg5OXVkmUREZsesVtG0xMfHB/fu3cP58+f11125cgWSJOH555/HM888g1u3bgEALly4gO7du8PR0VGucomIzIbCnM4muX79emRmZqKkpASOjo7o0aMHjh07hitXrmDjxo14+PAhGhoa8Oyzz2LXrl1QKpXIzs5GXFwc6uvrYWdnh1WrVmHo0KFy3xUiItmZVcATEZHxmH2LhoiI2ocBT0QkKLNaRVNeXgOdjh0jY3B2tkdpabXcZRA1w+em8SiVCjg6dn3sdrMKeJ1OYsAbEX+XZK743OwYbNEQEQmKAU9EJCgGPBGRoMyqB09E8tNqG1FeXozGxnqTzF9UpIROpzPJ3KKysrKBo6MLVKq2RTYDnoiaKC8vhq3tM+ja1R0KhcLo81tZKdHYyIA3lCRJqKmpRHl5MXr2VLdpX7ZoiKiJxsZ6dO3azSThTm2nUCjQtWu3dv2PigFPRM0w3M1Lex8PtmiI6IkcutnBtovxo6Lup0ZUVdYafV76NwZ8G5nqyW4KLi4OcpfQKr7IzZ9tFysELU02+rwpm4NRZcC4xMQPcfr0SWg0Bdi//0v07dtfv23q1CDY2NjAxqYLACAiYhFeeun/Npvj4cMKrFy5BHV1dRg3bjxCQ2e3q+aPP96N2bPnwtraul37d7TOkVRmxFRPdktl6IucLJef32uYNu13eOed8Ba3r18f3yT0W/L999/BwcEBu3Z98lS17Nu3FzNmzGpzwDc2NsLKquPjlgFPRGbt178e9lT7X7r0PbZv/wg//liDN98MRVTUcvTv74mEhK24c+cW6uvr4ePji0WLoqBSqXDgQBJOnMiEVtsIG5suWLYsGp6eXti8OR4AEBExFwqFEgkJu/Huu0sxY8Ys/OY3fgCAhQsX6C8vXLgAnp5euHo1G926dcOmTduQlPQpTp8+Ca1Wi549XbFy5So4O/fE2bNfY+/enVAqVdBqGxEVtQIvvOD71L87BjwRdWpxcWsASBgyZBjeeusdODg0bU2+8IIv5s//L/z972exfv0fAQB/+MM6DBv2AqKj10Cn0yEubjWOHfsKkyZNwfjxEzFjxkwAwIUL32Ljxg+wZ8+nWLp0JY4cOYSdOz/BM888Y1BtBQV52LHjT7CyskJGRhry8/Oxe/enUCqVOHLkMBITP0Rs7Hr86U+7sWLFKjz//FBotVrU1RmnbcmAJ6JOa/v2vXBzc0d9fT22bduMrVv/iJiYda3ul5V1BteuXcWXXz76rue6ujq4uroBAG7cuIbPP9+HysqHUCqV+Ne/7re7vrFjx+tbM1lZZ3D9+jXMnfvoj4dW2wh7e3sAwPDhvti2bQtee200Ro58udWWk6EY8ETUabm5uQMAbGxsMGXKNERHLzFwTwkbNmxC794eTa5taGjAmjUrkZi4F15eA1FSUozJk//zsbOoVFaQpH9/aKu+vuladTu7fx/pS5KEN96Yi8DA4GbzREYuxZ07t3Hx4gWsWRONkJAwTJo0xcD78nhcB09EnVJtbS2qqx+dV16SJBw/noH+/QcYtO9vfvMKkpI+g1arBQBUVFSgoCAf9fU/QavV6o/m//KXQ032e+aZrqip+fe57D08PHDtWg4A4O7df+L27ZuPvc1Ro17BkSOHUVlZCeDRH4Nbtx6Nv38/F/369cf06TMwbtx/6ud8WgYdwcfHxyMjIwP5+flISUnBgAHNf4nbt29HWloalEolrK2tERUVBT8/P6MUSUTyqfupESmbmx91GmNeQ3z44UacPn0KZWWlWLz4HXTr1h1JSf8PZWWlWL16BXQ6HbRaHfr0+RWWLo02aM7//u+l2LFjG958cwYUCgWsrW0QGbkUvXr1xrx5byE8fDa6deuO//iPMU32+93vwhAZ+V/o0sUWCQm7ERo6G2vWROPs2a8xYMBAeHp6PfY2x4+fiIcPK7Bo0QIAgE6nw5Qp0+DpOQA7dyYiL+8+VCor2Nvb4913Ywy6H60x6Eu3v//+e/Tu3RthYWHYtWtXiwF/9uxZ+Pr6ws7ODtevX8fMmTORlZUFW1tbg4spLa02+y8CcHFx4DJJI0rZHIziYi6UNCcPHtyDu/v/Mdn8PBdN+7T0uCiVCjg72z92H4OO4H19W1+u88ujdS8vL0iShIqKCri7uxtyE0REZGQmeZP16NGjeO6559oc7k/6S0Ti6gyfuLUkRUVKWFmZ9u05U88vIqVS2ebXitED/rvvvsNHH32ETz5p+yfGOkuLhoyLLRrzotPp0NCgNdkJx9iiaTtJkqDT6Zq9Vlpr0Rj1z+jly5exfPlybN++HX379jXm1ETUQX7+NCWZD622EUqlqs37GS3gr1y5gqioKGzbtg2DBw821rRE1MHs7OxRVVXRZH03yUeSdKiqKoedXdtb2Aa1aNavX4/MzEyUlJRgzpw56NGjB44dO4bw8HBERkZiyJAhiIuLQ11dHWJi/r28549//CO8vB6/bIiIzI+9fXeUlxejsDAPgPFbpkolv7KvbRSwsbGFvX33tu9pyDLJjtJZevBcJmk8XCZpeVxcHPiYG0mH9uCJiMh8MOCJiATFgCciEhQDnohIUAx4IiJBMeCJiATFgCciEhQDnohIUAx4IiJBMeCJiATFgCciEhQDnohIUAx4IiJBMeCJiATFgCciEhQDnohIUAx4IiJBMeCJiATFgCciEhQDnohIUAx4IiJBMeCJiATFgCciElSrAR8fH4/Ro0fDy8sLN2/ebHGMVqtFXFwc/P39MXbsWBw6dMjohRIRUdu0GvBjxozBF198gd69ez92TEpKCu7fv4/MzEwcPHgQCQkJyMvLM2qhRETUNq0GvK+vL9Rq9RPHpKWlYdq0aVAqlXBycoK/vz/S09ONViQREbWdlTEm0Wg06NWrl/6yWq3GgwcP2jyPs7O9McqhTsbFxUHuEqiD8THvGEYJeGMpLa2GTifJXcYT8YlpfMXFVXKXQB3IxcWBj7mRKJWKJx4YG2UVjVqtRkFBgf6yRqOBu7u7MaYmIqJ2MkrAjx8/HocOHYJOp0NZWRmOHz+OgIAAY0xNRETt1GrAr1+/Hq+88goePHiAOXPmYOLEiQCA8PBwZGdnAwCCg4Ph4eGBcePGYfr06XjnnXfw7LPPmrZyIiJ6IoUkSWbT9O4sPfigpclylyGMlM3B7MdaGPbgjadDevBERGR+GPBERIJiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKCtDBt29exfR0dGoqKhAjx49EB8fjz59+jQZU1painfffRcajQaNjY146aWXsHr1alhZGXQTRERkZAYdwcfGxiI0NBQZGRkIDQ1FTExMszG7du1Cv379kJKSgq+++gpXr15FZmam0QsmIiLDtBrwpaWlyMnJQWBgIAAgMDAQOTk5KCsrazJOoVCgpqYGOp0O9fX1aGhogJubm2mqJiKiVrXaP9FoNHBzc4NKpQIAqFQquLq6QqPRwMnJST/u7bffxqJFizBq1CjU1tYiLCwMw4cPb1Mxzs72bSyfRODi4iB3CdTB+Jh3DKM1yNPT0+Hl5YXPPvsMNTU1CA8PR3p6OsaPH2/wHKWl1dDpJGOVZBJ8YhpfcXGV3CVQB3JxceBjbiRKpeKJB8attmjUajUKCwuh1WoBAFqtFkVFRVCr1U3GJSUlYdKkSVAqlXBwcMDo0aPx7bffPmX5RETUXq0GvLOzM7y9vZGamgoASE1Nhbe3d5P2DAB4eHjgzJkzAID6+np888038PT0NEHJRERkCINW0axduxZJSUkICAhAUlIS4uLiAADh4eHIzs4GALz33nu4ePEigoKCMHnyZPTp0wfTp083XeVERPRECkmSzKbp3Vl68EFLk+UuQxgpm4PZj7Uw7MEbz1P34ImIqHNiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EZGgDAr4u3fvIiQkBAEBAQgJCUFubm6L49LS0hAUFITAwEAEBQWhpKTEmLUSEVEbWBkyKDY2FqGhoQgODkZycjJiYmKwf//+JmOys7ORmJiIzz77DC4uLqiqqoKNjY1JiiYiota1egRfWlqKnJwcBAYGAgACAwORk5ODsrKyJuM+/fRTzJ07Fy4uLgAABwcHdOnSxQQlExGRIVo9gtdoNHBzc4NKpQIAqFQquLq6QqPRwMnJST/uzp078PDwQFhYGH788UeMHTsWERERUCgUBhfj7GzfjrtAnZ2Li4PcJVAH42PeMQxq0RhCq9Xixo0b2LdvH+rr6zF//nz06tULkydPNniO0tJq6HSSsUoyCT4xja+4uEruEqgDubg48DE3EqVS8cQD41ZbNGq1GoWFhdBqtQAeBXlRURHUanWTcb169cL48eNhY2MDe3t7jBkzBleuXHnK8omIqL1aDXhnZ2d4e3sjNTUVAJCamgpvb+8m7RngUW8+KysLkiShoaEB58+fx8CBA01TNRERtcqgZZJr165FUlISAgICkJSUhLi4OABAeHg4srOzAQATJ06Es7MzJkyYgMmTJ6N///6YOnWq6SonIqInUkiSZDZN787Sgw9amix3GcJI2RzMfqyFYQ/eeJ66B09ERJ0TA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgEZVDA3717FyEhIQgICEBISAhyc3MfO/af//wnfv3rXyM+Pt5YNRIRUTsYFPCxsbEIDQ1FRkYGQkNDERMT0+I4rVaL2NhY+Pv7G7VIIiJqu1YDvrS0FDk5OQgMDAQABAYGIicnB2VlZc3G7tmzB6+99hr69Olj9EKJiKhtrFoboNFo4ObmBpVKBQBQqVRwdXWFRqOBk5OTftz169eRlZWF/fv3Y8eOHe0qxtnZvl37Uefm4uIgdwnUwfiYd4xWA94QDQ0NWLNmDT744AP9H4L2KC2thk4nGaMkk+ET0/iKi6vkLoE6kIuLAx9zI1EqFU88MG414NVqNQoLC6HVaqFSqaDValFUVAS1Wq0fU1xcjPv372PBggUAgMrKSkiShOrqaqxbt84Id4OIiNqq1YB3dnaGt7c3UlNTERwcjNTUVHh7ezdpz/Tq1Qvffvut/nJCQgJ+/PFHrFy50jRVExFRqwxaRbN27VokJSUhICAASUlJiIuLAwCEh4cjOzvbpAUSEVH7KCRJMpumd2fpwQctTZa7DGGkbA5mP9bCsAdvPK314PlJViIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFAMeCIiQTHgiYgExYAnIhIUA56ISFBWhgy6e/cuoqOjUVFRgR49eiA+Ph59+vRpMmb79u1IS0uDUqmEtbU1oqKi4OfnZ4qaiYjIAAYFfGxsLEJDQxEcHIzk5GTExMRg//79TcYMHToUc+fOhZ2dHa5fv46ZM2ciKysLtra2JimciIierNUWTWlpKXJychAYGAgACAwMRE5ODsrKypqM8/Pzg52dHQDAy8sLkiShoqLCBCUTEZEhWj2C12g0cHNzg0qlAgCoVCq4urpCo9HAycmpxX2OHj2K5557Du7u7m0qxtnZvk3jSQwuLg5yl0AdjI95xzCoRdMW3333HT766CN88sknbd63tLQaOp1k7JKMik9M4ysurpK7BOpALi4OfMyNRKlUPPHAuNUWjVqtRmFhIbRaLQBAq9WiqKgIarW62djLly9j+fLl2L59O/r27fsUZRMR0dNqNeCdnZ3h7e2N1NRUAEBqaiq8vb2btWeuXLmCqKgobNu2DYMHDzZNtUREZDCD1sGvXbsWSUlJCAgIQFJSEuLi4gAA4eHhyM7OBgDExcWhrq4OMTExCA4ORnBwMG7cuGG6yomI6IkUkiSZTdO7s/Tgg5Ymy12GMFI2B7Mfa2HYgzeep+7BExFR58SAJyISFAOeiEhQDHgiIkEx4ImIBMWAJyISFAOeiEhQDHgiIkEx4ImIBMWAJyISFAOeiEhQDHgiIkEx4ImIBMWAJyISFAOeiEhQDHgiIkEx4ImIBMWAJyISFAOeiEhQDHgiIkEx4ImIBMWAJyISlJXcBRCRcTh0s4Ntl87xknZxcZC7hFbV/dSIqspauct4Kp3j2UBErbLtYoWgpclylyGMlM3BqJK7iKdkUIvm7t27CAkJQUBAAEJCQpCbm9tsjFarRVxcHPz9/TF27FgcOnTI2LUSEVEbGBTwsbGxCA0NRUZGBkJDQxETE9NsTEpKCu7fv4/MzEwcPHgQCQkJyMvLM3rBRERkmFZbNKWlpcjJycG+ffsAAIGBgVi3bh3Kysrg5OSkH5eWloZp06ZBqVTCyckJ/v7+SE9Px/z58w0uRqlUtOMudDxXRzu5SxBKZ3ncOwM+N43L3J+brdXXasBrNBq4ublBpVIBAFQqFVxdXaHRaJoEvEajQa9evfSX1Wo1Hjx40KZiHR27tmm8XD5ePU7uEoTi7GwvdwnC4HPTuDr7c5PLJImIBNVqwKvVahQWFkKr1QJ49GZqUVER1Gp1s3EFBQX6yxqNBu7u7kYul4iIDNVqwDs7O8Pb2xupqakAgNTUVHh7ezdpzwDA+PHjcejQIeh0OpSVleH48eMICAgwTdVERNQqhSRJUmuD7ty5g+joaFRWVqJbt26Ij49H3759ER4ejsjISAwZMgRarRbvv/8+zp07BwAIDw9HSEiIye8AERG1zKCAJyKizodvshIRCYoBT0QkKAY8EZGgGPBERIJiwBMRCYoBT0QkKAY8EXWIyspK1NTUyF2GReEXfhCRyVRWVmLz5s1ISUlBbe2jb0dyc3PDvHnzMGvWLJmrEx8/6CSAhw8fonv37vrLhw8fxqVLl+Dt7Y2ZM2dCoTDvU56SuCIiIjB06FC8+uqrSElJgaOjI0aOHImEhAQMGTIEkZGRcpcoNLZoBPDmm2/qf/74449x6NAhDBo0CKdOncKWLVvkK4wsXl5eHiIiIjBo0CCsXLkSJ06cwNChQ5GYmIhjx47JXZ7wGPAC+OV/wtLS0rBz507MnDkT27dvx6lTp2SsjCydQqFARUUFACA/Px86nQ4A0KVLF1hZsUNsavwNC+CXLRiFQqE/06ednR1fRCSrN954A5MmTcKgQYOQnZ2NVatWAQBKSkqafEEQmQZ78AIYPHgwunXrBkmSUFNTg9OnT8PJyQmNjY2YMmUKUlJS5C6RLNidO3dw69YtDBw4EH369JG7HIvCgBdAfn5+k8uurq6wtrZGWVkZLl68iLFjx8pUGRHJiQFPRCZTUFCADz74AEqlEqtXr8aOHTuQnJwMLy8vbNy4ER4eHnKXKDS+ySqAyMhI/O1vf0NjY6PcpRA1ERsbixdffBFeXl6YO3cu3N3dkZmZiQkTJmDDhg1ylyc8HsELYOTIkfrvzg0KCsLrr7+OAQMGyF0WEYKDg5GcnAwAGDVqFLKysvTbJk2ahK+++kqu0iwCj+AF4O7ujiNHjmDv3r1obGzErFmzMHXqVBw4cADV1dVyl0cW7JcrvAYNGvTYbWQaDHgB/PxCGTx4MNasWYOzZ89izpw5OH78OPz8/GSujiyZra2t/iBjz549+uvLy8uhUqnkKstisEUjgMmTJ+Po0aMtbnvw4AHc3d07uCKiRyRJavFIvaysDCUlJWwlmhgDXgAZGRkICAiQuwwiMjNs0QiA4U6d0fz58+UuQXgMeAFwmSSZq9ra2sf+u3XrltzlCY8tGgFwmSSZq4EDB0KhUDQ5Id7PlxUKBa5duyZjdeLjmagE8PMyyatXr+Ivf/kLZs2ahWeffRavv/46goKCYG9vL3eJZKFcXFyQnJysPwHeL7366qsyVGRZ2KIRAJdJkrl66aWXHtuKGTp0aAdXY3nYohEAl0kSUUt4BC+AiIiIx25juBNZLga8ALhMkjojLpM0Pb7JKoCGhgbs27cPqampKCgogEqlQv/+/TFv3jyMHj1a7vLIgtXW1j52G5dJmh4DXgAxMTGwsrLCokWL8Ne//hV9+vSBp6cnEhMTodFoEBYWJneJZKF8fHyeuEySTItvsgpgwoQJSEtLAwD92SQPHDiAhw8fIiQkBOnp6TJXSJbKz8/vicskT58+LUNVloM9eAGoVCr89NNPAIDq6mr9z927d4e1tbWcpZGF4zJJebFFI4DAwEBMnz4dvr6+OHfunL4lU1JSInNlZOk2bdrU7Lq///3vePnll5GQkCBDRZaFLRpBZGVl4caNGxg8eDBGjhwpdzlEAIDbt283u27evHn45JNPIEkS+vfvL0NVloMBT0QmM3DgQPTu3bvJm6yFhYVwc3ODQqHAiRMnZKxOfOzBC6C+vh47d+7EmjVr8PXXXzfZtm7dOnmKIgKwcOFC9O3bF0lJSTh58iROnjwJNzc3nDx5kuHeARjwAli7di1u3ryJvn37YtOmTfj973+v33bp0iUZKyNLt3DhQkRFRWHJkiU4cOAAAH4Xa0diwAsgOzsbW7duxZw5c3D48GHk5+fjvffegyRJYAeO5DZo0CDs378f+fn5ePPNN9HQ0CB3SRaDAS8ArVar/9nW1hYJCQmora3F8uXLodPpZKyM6BEbGxssW7YMixcvxsyZM+Uux2Iw4AXQs2dPXL9+XX9ZpVJh8+bNUCgU/Dg4mZVhw4ZhwYIFcpdhMbiKRgC5ubmwtrZG7969m1wvSRLOnDnDL1YgslAMeCIiQbFFI4Bz587pf66qqsLy5cvh7++PRYsW8dOsRBaMAS+AX34cfOvWrejatSt27NiBvn37Yv369TJWRkRy4rloBPDLLtvFixdx+PBhWFtbY8CAAQgKCpKxMiKSEwNeAPX19bhz547+HNu/PIOkUsn/pBFZKga8AOrq6rBgwQL9kfzP5/qorq5mwBNZMK6iEVhtbS1KS0vh4eEhdylEJAMe3gnMzs4OERERcpdBRDJhi0YALZ1z+2fl5eUdWAkRmRMGvAACAwObnXP7ZxUVFTJURETmgAEvgN69e+PPf/4z3Nzcmm3jaQqILBd78AIYN24c8vPzW9w2duzYDq6GiMwFV9EQEQmKR/BERIJiwBMRCYoBTxajpKQEYWFh8PHxwR/+8Ae5yyEyOa6iIbNVX1+PtWvX4ptvvkFFRQWee+45LFmyRL8y6B//+Ac++ugjXL16FUqlEiNGjMDq1avh6ura4nwHDx6Eo6MjLl269NRf/BwdHQ03NzdERUU91TxEpsQjeDJbjY2NUKvV+Pzzz3Hx4kUsXrwYixcvRl5eHgDg4cOHmD59Ok6ePIlTp06ha9euePfddx87X0FBAfr16/fU4W4MjY2NcpdAlkAi6kQCAwOl9PT0Frf9z//8jzRs2LAWt61cuVIaNGiQNHjwYGnYsGHSuXPnJK1WK+3evVsaM2aMNGLECCkyMlIqLy/X77No0SLp5Zdfll544QUpNDRUunnzpiRJkvTll182meutt96SJEmSBgwYIOXm5ja5zS1btkiSJEnnz5+X/Pz8pN27d0svv/yytGzZsifefl1dnbR06VJpxIgR0vDhw6Xf/va3UnFx8dP/AsmisEVDnUZJSQlyc3PRv3//FrdfuHABnp6eLW77uef+y7bKZ599huPHjyMpKQlOTk5Yv3493n//fWzZsgUA8Morr2DDhg2wsbHBxo0bsWzZMiQnJyMkJASXL19uc4umpKQEDx8+xKlTp6DT6fD5558/9vaPHDmC6upqfP3117CxscG1a9dga2vbll8XEVs01Dk0NDRg2bJlmDJlCvr169ds+/Xr17Fjxw6sWLHC4Dm//PJLREVFwd3dHTY2Nli4cCEyMjL07ZOpU6fC3t4eNjY2WLRoEa5fv46qqqp23welUonIyEjY2NjA1tb2ibdvZWWFiooK3Lt3DyqVCs8//zzs7e3bfdtkmXgET2ZPp9NhxYoVsLa2xpo1a5ptv3fvHsLDw/Hee+/B19fX4HkLCgrwzjvvNDlnvlKpRGlpKXr27ImtW7ciPXj1dXwAAAIBSURBVD0dZWVl+jHl5eVwcHBo1/1wdHREly5dDLr94OBgPHjwAEuWLEFlZSUmTZqEqKioJl/mQtQaBjyZNUmSsGrVKpSUlGDv3r3NAi4/Px9z5szB22+/jcmTJ7dpbnd3d2zYsAHDhw9vtu3o0aM4ceIE9u3bBw8PD1RVVeHFF1/Un9CtpTdq7ezsUFtbq79cXFzc5PxA//8+T7p9AFi4cCEWLlyIvLw8LFiwAL/61a8wbdq0Nt1Hsmxs0ZBZi42NxZ07d7Br165mPejCwkK88cYbCAsLw4wZM9o894wZM/Dhhx/qz+NTVlaG48ePAwBqampgY2MDR0dH1NbW6vvyP3N2dtav5vnZwIEDkZqaCq1WizNnzuDChQvtvv3z58/jxo0b0Gq1sLe3h5WVFb+di9qMzxgyW/n5+Th48CCuXbuGUaNGwcfHBz4+Pvjqq68AAIcOHcK//vUvJCYm6rf5+PgYPP/s2bMxevRozJ07Fz4+Ppg+fTquXLkCAJg8eTJ69eoFPz8/TJw4EcOGDWuy79SpU3H79m34+vri7bffBgCsWrUKp06dgq+vL1JSUuDv79/u2y8pKUFkZCSGDx+OCRMmYMSIEQgODjb4vhEBPNkYEZGweARPRCQoBjwRkaAY8EREgmLAExEJigFPRCQoBjwRkaAY8EREgmLAExEJ6n8BEK0W0+YibwgAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "8iCWXD3uY_lI", + "outputId": "8402101d-1ad7-41d2-a2bf-702ddfe023c9" + }, + "source": [ + "!python --version\n" + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Python 3.7.11\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Y-OpZfs184pn" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file From e129069b039f746196bec8ad02e490d2f6f38497 Mon Sep 17 00:00:00 2001 From: Quanghuy99 <30777550+Quanghuy99@users.noreply.github.com> Date: Sat, 14 Aug 2021 19:32:15 +0700 Subject: [PATCH 2/2] Created using Colaboratory --- Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb b/Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb index 01d3294..3100d82 100644 --- a/Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb +++ b/Copy_of_HACKANONS_COLAB_25GB_RAM.ipynb @@ -35,7 +35,7 @@ "base_uri": "https://localhost:8080/" }, "id": "EN-xy98G3C95", - "outputId": "14450904-f512-483e-dd4e-e9ce341c1475" + "outputId": "51026caa-f8d7-4ba8-9821-5693684657f5" }, "source": [ "from google.colab import drive\n",