ranked_retrieval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from load_data import LoadDataset \n",
    "from vsm import VectorSpaceModel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the dataset\n",
    "Functions to load the dataset and the given queries with associated relevant documents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = LoadDataset(\"./med_data/MED.ALL\", \"./med_data/MED.QRY\", \"./med_data/MED.REL\")\n",
    "\n",
    "docs = dataset.load_docs()\n",
    "queries = dataset.load_queries()\n",
    "relevance = dataset.load_relevance()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vector Space Model ranked retrieval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the `vsm()` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "vsm = VectorSpaceModel(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents in the collection:  1032\n",
      "Number of terms in the collection:  9685\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of documents in the collection: \", vsm.n_docs)\n",
    "print(\"Number of terms in the collection: \", vsm.n_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The inverted index of the collection: first 10 terms and first 10 documents per term\n",
      "Term:  correl \tDocs:  [0, 385, 904, 905, 394, 911, 147, 148, 277, 153] \n",
      "\n",
      "Term:  matern \tDocs:  [0, 4, 5, 11, 303, 706, 324, 326, 328, 331] \n",
      "\n",
      "Term:  fetal \tDocs:  [0, 1, 2, 3, 4, 5, 904, 11, 277, 936] \n",
      "\n",
      "Term:  plasma \tDocs:  [0, 1024, 4, 5, 516, 1031, 397, 273, 147, 149] \n",
      "\n",
      "Term:  level \tDocs:  [0, 1, 1024, 3, 516, 517, 9, 523, 524, 525] \n",
      "\n",
      "Term:  glucos \tDocs:  [0, 640, 4, 518, 9, 146, 413, 297, 564, 181] \n",
      "\n",
      "Term:  free \tDocs:  [0, 385, 3, 4, 393, 11, 652, 275, 659, 281] \n",
      "\n",
      "Term:  fatti \tDocs:  [0, 386, 4, 5, 6, 7, 9, 137, 157, 158] \n",
      "\n",
      "Term:  acid \tDocs:  [0, 1, 4, 5, 6, 7, 520, 9, 1030, 526] \n",
      "\n",
      "Term:  coeffici \tDocs:  [0, 40, 305, 401, 532, 501, 502, 628, 507] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"The inverted index of the collection: first 10 terms and first 10 documents per term\")\n",
    "for t, v in list(vsm.index.items())[:10]:\n",
    "    print(\"Term: \", t, \"\\tDocs: \", list(v)[:10], \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualize the documents as vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_vector = vsm.docs_as_vectors()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1032, 9685)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs_vector.shape "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualize the queries as vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example query:   the crystalline lens in vertebrates including humans\n"
     ]
    }
   ],
   "source": [
    "example_query = queries[1]\n",
    "print(\"Example query: \", example_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_vector = vsm.query_as_vector(example_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9685,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_vector.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute the relevance scores as the cosine similarity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = vsm.relevance_scores(example_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The relevance scores for the given query, for each document:\n",
      "DocID:  0 \tScore:  0 \n",
      "\n",
      "DocID:  1 \tScore:  0 \n",
      "\n",
      "DocID:  2 \tScore:  0 \n",
      "\n",
      "DocID:  3 \tScore:  0.021291796307282396 \n",
      "\n",
      "DocID:  4 \tScore:  0 \n",
      "\n",
      "DocID:  5 \tScore:  0 \n",
      "\n",
      "DocID:  6 \tScore:  0 \n",
      "\n",
      "DocID:  7 \tScore:  0 \n",
      "\n",
      "DocID:  8 \tScore:  0.056132485052834566 \n",
      "\n",
      "DocID:  9 \tScore:  0 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"The relevance scores for the given query, for each document:\")\n",
    "for d, s in list(scores.items())[:10]:\n",
    "    print(\"DocID: \", d, \"\\tScore: \", s, \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Return the top 10 documents given the query."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' the crystalline lens in vertebrates including humans'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "k = 10\n",
    "top10 = vsm.vector_space_model(example_query, k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The relevance scores for the top 10 documents:\n",
      "DocID:  71 \tScore:  0.32326427430871757 \n",
      "\n",
      "DocID:  499 \tScore:  0.2662094450787526 \n",
      "\n",
      "DocID:  180 \tScore:  0.2179920012480751 \n",
      "\n",
      "DocID:  170 \tScore:  0.19139097770770058 \n",
      "\n",
      "DocID:  14 \tScore:  0.19027283535863754 \n",
      "\n",
      "DocID:  964 \tScore:  0.18674186882310087 \n",
      "\n",
      "DocID:  165 \tScore:  0.18475982417434741 \n",
      "\n",
      "DocID:  512 \tScore:  0.17601476449236989 \n",
      "\n",
      "DocID:  137 \tScore:  0.16698722946249522 \n",
      "\n",
      "DocID:  359 \tScore:  0.1503014631326245 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(f\"The relevance scores for the top {k} documents:\")\n",
    "for d, s in list(top10.items()):\n",
    "    print(\"DocID: \", d, \"\\tScore: \", s, \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Relevance Feedback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Relevant documents for the example query:  [12, 13, 14, 71, 78, 137, 141, 163, 164, 165, 166, 167, 168, 169, 170, 171, 179, 180, 181, 182, 183, 184, 185, 210, 211, 498, 499, 500, 501, 502, 503, 505, 506, 507, 509, 510, 512]\n"
     ]
    }
   ],
   "source": [
    "example_relevant = relevance[1]\n",
    "print(\"Relevant documents for the example query: \", example_relevant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 of the non relevant documents for the example query:  [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15, 16, 17, 18, 19, 20, 21, 22]\n"
     ]
    }
   ],
   "source": [
    "example_non_relevant = [i for i in range(vsm.n_docs) if i not in example_relevant]\n",
    "print(\"20 of the non relevant documents for the example query: \", example_non_relevant[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_query = vsm.relevance_feedback_rocchio(example_query, example_relevant, example_non_relevant, alpha=1, beta=.75, gamma=.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "top10_rel = vsm.vector_space_model(opt_query, k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The relevance scores for the top 10 documents retrieved using relevance feedback:\n",
      "DocID:  498 \tScore:  0.4975825553600744 \n",
      "\n",
      "DocID:  180 \tScore:  0.48248989126391234 \n",
      "\n",
      "DocID:  510 \tScore:  0.472073021470223 \n",
      "\n",
      "DocID:  12 \tScore:  0.4560654940412232 \n",
      "\n",
      "DocID:  179 \tScore:  0.45186607965425535 \n",
      "\n",
      "DocID:  508 \tScore:  0.44791318658586426 \n",
      "\n",
      "DocID:  164 \tScore:  0.42581831309501617 \n",
      "\n",
      "DocID:  165 \tScore:  0.42358178294680243 \n",
      "\n",
      "DocID:  499 \tScore:  0.4230008437357246 \n",
      "\n",
      "DocID:  503 \tScore:  0.4203885699026642 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(f\"The relevance scores for the top {k} documents retrieved using relevance feedback:\")\n",
    "for d, s in list(top10_rel.items()):\n",
    "    print(\"DocID: \", d, \"\\tScore: \", s, \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pseudo-Relevance Feedback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_query_pseudo = vsm.pseudo_relevance_feedback(example_query, k=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "top10_pseudo = vsm.vector_space_model(opt_query_pseudo, k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The relevance scores for the top 10 documents retrieved using pseudo relevance feedback:\n",
      "DocID:  14 \tScore:  0.5571136148407742 \n",
      "\n",
      "DocID:  165 \tScore:  0.5392188837587554 \n",
      "\n",
      "DocID:  512 \tScore:  0.5216182443784564 \n",
      "\n",
      "DocID:  499 \tScore:  0.48581512098078977 \n",
      "\n",
      "DocID:  180 \tScore:  0.46169998787542205 \n",
      "\n",
      "DocID:  170 \tScore:  0.4382538945134972 \n",
      "\n",
      "DocID:  71 \tScore:  0.4340505106725038 \n",
      "\n",
      "DocID:  964 \tScore:  0.3458494017629007 \n",
      "\n",
      "DocID:  137 \tScore:  0.3081767502378574 \n",
      "\n",
      "DocID:  12 \tScore:  0.30238538545091015 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(f\"The relevance scores for the top {k} documents retrieved using pseudo relevance feedback:\")\n",
    "for d, s in list(top10_pseudo.items()):\n",
    "    print(\"DocID: \", d, \"\\tScore: \", s, \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performance evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Computing precision and recall for the given set of queries and relevant documents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "map = dict()\n",
    "prec = dict()\n",
    "rec = dict()\n",
    "\n",
    "min_k = 1\n",
    "max_k = max([len(i) for i in relevance.values()])\n",
    "prec, rec = vsm.precision_recall(queries, relevance, min_k, max_k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "prec_rel, rec_rel = vsm.precision_recall(queries, relevance, min_k, max_k, rel_feedback=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "prec_pseudo, rec_pseudo = vsm.precision_recall(queries, relevance, min_k, max_k, pseudo_feedback=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average precision and recall and Mean Average Precision (MAP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def max_avg_prec(precision, queries, relevance):\n",
    "    map = 0\n",
    "    for qid in list(queries.keys()):\n",
    "        ap = 0\n",
    "        mj = len(relevance[qid])\n",
    "        for k in range(1, mj+1):\n",
    "            ap += precision[k][qid-1]\n",
    "        map += ap / mj\n",
    "    map /= len(queries.keys())\n",
    "    return map"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. For the \"standard\" case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K=5\tAverage Precision: 0.6827586206896552\t Average Recall: 0.16281932949952388\n",
      "K=6\tAverage Precision: 0.6551724137931034\t Average Recall: 0.19058144148202566\n",
      "K=7\tAverage Precision: 0.6403940886699508\t Average Recall: 0.21561975105582612\n",
      "K=8\tAverage Precision: 0.6422413793103449\t Average Recall: 0.2472395322148463\n",
      "K=9\tAverage Precision: 0.6513409961685824\t Average Recall: 0.2792880888851007\n",
      "K=10\tAverage Precision: 0.6413793103448275\t Average Recall: 0.3074866616191644\n",
      "K=11\tAverage Precision: 0.6175548589341692\t Average Recall: 0.32182840087469666\n",
      "K=12\tAverage Precision: 0.6091954022988506\t Average Recall: 0.3435653605129643\n",
      "K=13\tAverage Precision: 0.5941644562334217\t Average Recall: 0.361982461771145\n",
      "K=14\tAverage Precision: 0.5837438423645319\t Average Recall: 0.3843476271535518\n"
     ]
    }
   ],
   "source": [
    "for k in range(5, 15):\n",
    "    print(f\"K={k}\\tAverage Precision: {np.mean(prec[k])}\\t Average Recall: {np.mean(rec[k])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "map = max_avg_prec(prec, queries, relevance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP: 0.6296379359580575\n"
     ]
    }
   ],
   "source": [
    "print(f\"MAP: {map}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Allowing relevance feedback - considering as relevant the known set of relevant documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K=5\tAverage Precision: 0.9586206896551723\t Average Recall: 0.23514480661536444\n",
      "K=6\tAverage Precision: 0.9540229885057472\t Average Recall: 0.28206434548418724\n",
      "K=7\tAverage Precision: 0.9458128078817735\t Average Recall: 0.32513901843917864\n",
      "K=8\tAverage Precision: 0.9482758620689655\t Average Recall: 0.37123503786379247\n",
      "K=9\tAverage Precision: 0.9425287356321839\t Average Recall: 0.41206285805468973\n",
      "K=10\tAverage Precision: 0.9344827586206896\t Average Recall: 0.45115531341052195\n",
      "K=11\tAverage Precision: 0.9247648902821317\t Average Recall: 0.49087010231597417\n",
      "K=12\tAverage Precision: 0.9137931034482759\t Average Recall: 0.5269035789194244\n",
      "K=13\tAverage Precision: 0.8992042440318303\t Average Recall: 0.5588493169523775\n",
      "K=14\tAverage Precision: 0.8866995073891624\t Average Recall: 0.5895345976903904\n"
     ]
    }
   ],
   "source": [
    "for k in range(5, 15):\n",
    "    print(f\"K={k}\\tAverage Precision: {np.mean(prec_rel[k])}\\t Average Recall: {np.mean(rec_rel[k])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_rel = max_avg_prec(prec_rel, queries, relevance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP allowing relevance feedback: 0.9321421215043031\n"
     ]
    }
   ],
   "source": [
    "print(f\"MAP allowing relevance feedback: {map_rel}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. Allowing pseudo-relevance feedback - considering as relevant the top 10 documents retrieved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K=5\tAverage Precision: 0.7793103448275862\t Average Recall: 0.19113517390258328\n",
      "K=6\tAverage Precision: 0.7586206896551723\t Average Recall: 0.22187928788908468\n",
      "K=7\tAverage Precision: 0.7389162561576356\t Average Recall: 0.24887198088278584\n",
      "K=8\tAverage Precision: 0.7112068965517241\t Average Recall: 0.27617769102957546\n",
      "K=9\tAverage Precision: 0.7011494252873562\t Average Recall: 0.30235656890286466\n",
      "K=10\tAverage Precision: 0.6896551724137929\t Average Recall: 0.32787548666316174\n",
      "K=11\tAverage Precision: 0.677115987460815\t Average Recall: 0.35559589840557104\n",
      "K=12\tAverage Precision: 0.6781609195402301\t Average Recall: 0.3871030980689513\n",
      "K=13\tAverage Precision: 0.6631299734748012\t Average Recall: 0.4105198274904371\n",
      "K=14\tAverage Precision: 0.6576354679802955\t Average Recall: 0.4341823353436082\n"
     ]
    }
   ],
   "source": [
    "for k in range(5, 15):\n",
    "    print(f\"K={k}\\tAverage Precision: {np.mean(prec_pseudo[k])}\\t Average Recall: {np.mean(rec_pseudo[k])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_pseudo = max_avg_prec(prec_pseudo, queries, relevance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP allowing pseudo-relevance feedback: 0.7072055147428038\n"
     ]
    }
   ],
   "source": [
    "print(f\"MAP allowing pseudo-relevance feedback: {map_pseudo}\")"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}