Finding_Problematic_URL_Parameters_through_Log_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Finding Problematic URL Parameters through Log Analysis",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/anirudh-tatavarthi/Twittorials/blob/master/Finding_Problematic_URL_Parameters_through_Log_Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uc16DtV1VKZY",
        "colab_type": "text"
      },
      "source": [
        "## Not all URL parameters show up in search console or the sitemaps. By studying the logs, you can find these parameters to locate duplicate content"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "91n8JAXpVhH_",
        "colab_type": "text"
      },
      "source": [
        "### This script will allow you to identify the many URL parameters present in your log files, and the number of times they appear on unique URLs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5F1crj11ZsCR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#This block of code installs the necessary packages\n",
        "%%capture\n",
        "import csv\n",
        "!pip install lars\n",
        "from lars.apache import ApacheSource\n",
        "import plotly.graph_objects as go"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FQQ7rqgRGQTD",
        "colab_type": "text"
      },
      "source": [
        "**Step 1:** When this block runs, you will be prompted to choose a file. Select a text file containing the logs."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6W8FPCPFyIj5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "\n",
        "#prompts user to choose a file\n",
        "uploaded = files.upload()\n",
        "\n",
        "#reads the name of the file\n",
        "for fn in uploaded.keys():\n",
        "  file_name = fn"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jtgau5Z1GfBE",
        "colab_type": "text"
      },
      "source": [
        "**Step 2:** This next block of code goes through the logs and gathers information on all of the parameters."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PdED6b_SGp_o",
        "colab_type": "text"
      },
      "source": [
        "At the end, you will be able to view a bar graph in order to visualize the data, as well as a download to view the data in an excel file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IJ-SODjYZx21",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import csv\n",
        "from lars.apache import ApacheSource\n",
        "\n",
        "#create sets storing parameters and urls\n",
        "paramset = dict()\n",
        "urlset = set()\n",
        "\n",
        "#open the file and identify the parameters with Apache Source\n",
        "with open(file_name) as f:\n",
        "    with ApacheSource(f) as source:\n",
        "        for row in source:\n",
        "          #if there is a parameter present\n",
        "          if(len(row.request.url.query_str)>0):\n",
        "             parameter = \"?\" + row.request.url.query_str\n",
        "             url = row.request.url.path_str+parameter\n",
        "             #finds all of the parameters in the query string\n",
        "             while (parameter.find('&')!=-1):\n",
        "                parameter.find('=')\n",
        "                sub = parameter[parameter.find('&')+1:parameter.find('=')]\n",
        "                if(len(sub)>0):\n",
        "                  #if a new one is found, it is added to the set\n",
        "                  if(sub not in paramset.keys()):\n",
        "                    paramset[sub]=0\n",
        "                parameter = parameter[parameter.find('=')+1:len(parameter)]\n",
        "             parameter = \"?\" + row.request.url.query_str\n",
        "             #if a new unique URL is found containing these parameters\n",
        "             if((url in urlset) ==False):\n",
        "              while (parameter.find('&')!=-1):\n",
        "                  parameter.find('=')\n",
        "                  sub = parameter[parameter.find('&')+1:parameter.find('=')]\n",
        "                  if(len(sub)>0):\n",
        "                    #update the count for the parameter \n",
        "                    paramset[sub]=paramset.get(sub)+1\n",
        "                  parameter = parameter[parameter.find('=')+1:len(parameter)]\n",
        "              #add the URL to a set of found URLs\n",
        "              urlset.add(url)\n",
        "        #sort dict from least to greatest\n",
        "        sort_orders = sorted(paramset.items(), key=lambda x: x[1])\n",
        "        paramset=dict(sort_orders)\n",
        "        #write the csv output file   \n",
        "        for k in paramset.keys():\n",
        "          output = \"[Parameter: \" + k + \" , count: \" + str(paramset[k]) + \" ]\"\n",
        "          with open('output.csv', 'a', newline='') as f:\n",
        "                thewriter = csv.writer(f)\n",
        "                thewriter.writerow([output])\n",
        "\n",
        "#download the output file\n",
        "files.download('output.csv')\n",
        "\n",
        "#sorts the dictionary of parameters from least appeared to most\n",
        "sort_orders = sorted(paramset.items(), key=lambda x: x[1], reverse=False)\n",
        "\n",
        "key_list = list(paramset.keys()) \n",
        "val_list = list(paramset.values()) \n",
        "\n",
        "#creates a horizontal bar graph with the data\n",
        "fig = go.Figure([go.Bar(x=val_list, y=key_list, orientation='h')])\n",
        "fig.show()\n",
        "\n",
        "#completion message\n",
        "print(\"Done!\")\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}