Code/Retrieval-Methods-and-Evaluation/Method-3-Multimodal-Retrieval.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "- 2D Search -  Freq [Liver, Colon, Pancreas, Lung]"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss \n",
        "import pandas as pd\n",
        "import torch\n",
        "from tqdm.notebook import tqdm\n",
        "import glob \n",
        "import pickle\n",
        "import numpy as np\n",
        "from random import sample\n",
        "import os\n",
        "import open_clip\n",
        "import csv\n",
        "from sklearn.metrics import average_precision_score\n",
        "import math\n",
        "####################################################\n",
        "## Select one of the organs: liver |colon | pancreas | lung\n",
        "organ=\"liver\"\n",
        "####################################################\n",
        "\n",
        "## Methods:\n",
        "method1 = 'biomedclip_caption-based' # caption-based search method\n",
        "method2 = 'biomedclip_caption+slice' # hybrid method combining the caption- and slice-based searches\n",
        "\n",
        "## Quantification Files\n",
        "quantification_dir = \"data/msd_final/quantification_file\"\n",
        "quantification_csv = quantification_dir + \"/msd_\" + organ + \"_metrics_3D_subject_including_normal_tissue.csv\"\n",
        "\n",
        "data_splits_folder = \"data/msd_train_test_split\"\n",
        "if (organ == \"lung\"):\n",
        "    data_splits_folder = \"data/msd_train_test_split/lung\"\n",
        "\n",
        "all_embs_folder = \"data/msd_embeddings/biomedclip_2d\"\n",
        "\n",
        "# Image Captions\n",
        "captionsFile=\"msd_gpt4_captions/gpt4_captions_msd_\"+organ+\".csv\" \n",
        "\n",
        "test_set_csv =  data_splits_folder + \"/\" + organ + \"/\" + organ + \"_full_test_split.csv\"\n",
        "## Output files \n",
        "results_dir = \"BiomedCLIP/MSD-Final/results/results_\" + organ\n",
        "results_eval_metrics = results_dir + \"/\" + organ + \"eval_metrics.csv\"\n",
        "\n",
        "## 2D Index for the slice-based and caption-based searches\n",
        "ids_organ_path = \"BiomedCLIP/MSD-Final/2d-index/\" + organ + \"-train-2d-index/faiss_ids_embd_train.tsv\"\n",
        "index_organ_path = \"BiomedCLIP/MSD-Final/2d-index/\" + organ + \"-train-2d-index/faiss_index_embd_train.index\"\n"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1700282090547
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class FAISSRetrievalEngine:\n",
        "    def __init__(self, ids_path, index_path):\n",
        "        self.ids_path = ids_path\n",
        "        self.ids = pd.read_csv(self.ids_path, sep=\"\\t\")\n",
        "        self.ids = {row[\"ids_int\"]: row[\"ids_str\"] for i, row in self.ids.iterrows()}\n",
        "\n",
        "        self.device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "        self.index_path = index_path\n",
        "        self.index = faiss.read_index(self.index_path)\n",
        "        if self.device == \"cuda\":\n",
        "            res = faiss.StandardGpuResources()\n",
        "            self.index = faiss.index_cpu_to_gpu(res, 0, self.index)\n",
        "    \n",
        "    def _split_array_into_batches(self, arr, batch_size):\n",
        "        num_rows = arr.shape[0]\n",
        "        num_batches = (num_rows + batch_size - 1) // batch_size  \n",
        "        \n",
        "        # Create a list of batches\n",
        "        batches = []\n",
        "        for i in range(num_batches):\n",
        "            start_idx = i * batch_size\n",
        "            end_idx = min((i + 1) * batch_size, num_rows)\n",
        "            batch = arr[start_idx:end_idx, :]\n",
        "            batches.append(batch)\n",
        "\n",
        "        return batches\n",
        "\n",
        "    def get_str_ids_from_I(self, I):\n",
        "        mapping_fn = lambda x: self.ids[x]\n",
        "        return np.vectorize(mapping_fn)(I)\n",
        "\n",
        "    def retrieve_images(self, xq, k=5):\n",
        "        batches = self._split_array_into_batches(xq, 16)\n",
        "        output_D, output_I = [], []\n",
        "        for batch in tqdm(batches, total=len(batches)):\n",
        "            D, I = self.index.search(batch, k)\n",
        "            I = self.get_str_ids_from_I(I)\n",
        "            output_D.append(D)\n",
        "            output_I.append(I)\n",
        "        output_D = np.concatenate(output_D, axis=0)\n",
        "        output_I = np.concatenate(output_I, axis=0)\n",
        "        return output_D, output_I\n",
        "\n",
        "class BiomedCLIPTextEmbeddingGenerator:\n",
        "    def __init__(self):\n",
        "        self.model, _, _ = open_clip.create_model_and_transforms(\n",
        "            'hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224'\n",
        "            )\n",
        "        self.tokenizer = open_clip.get_tokenizer(\n",
        "            'hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224'\n",
        "            )\n",
        "        self.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "        self.model.to(self.device)\n",
        "    \n",
        "\n",
        "    def __call__(self, phrases):\n",
        "        if not isinstance(phrases, list):\n",
        "            phrases = [phrases]\n",
        "        \n",
        "        texts = self.tokenizer(phrases, context_length=256).to(self.device)\n",
        "        with torch.no_grad():\n",
        "            _, text_features, _ = self.model(None, texts)\n",
        "        \n",
        "        return text_features.cpu().detach().numpy()\n",
        "\n",
        "class Text2ImageRetrieval: \n",
        "    def __init__(self, ids_path, index_path):        \n",
        "        self.index = FAISSRetrievalEngine(ids_path, index_path)\n",
        "        print(\" (1/2) index: done\")\n",
        "        self.model = BiomedCLIPTextEmbeddingGenerator()\n",
        "        print(\"(2/2) model: done\")\n",
        "    \n",
        "    def __call__(self, phrases, k=4):\n",
        "        embd = self.model(phrases)\n",
        "        embd_norm = embd / np.linalg.norm(embd, axis=1)[:, np.newaxis]\n",
        "        D, I = self.index.retrieve_images(embd_norm, 20) ## 20 slices \n",
        "        return D, I\n",
        "\n",
        "#######################################\n",
        "## Image2ImageRetrieval\n",
        "index = FAISSRetrievalEngine(\n",
        "    ids_path=ids_organ_path,\n",
        "    index_path=index_organ_path\n",
        ")\n",
        "\n",
        "## Text2ImageRetrieval\n",
        "ret_engine = Text2ImageRetrieval(\n",
        "    ids_path=ids_organ_path,\n",
        "    index_path=index_organ_path\n",
        ")\n",
        "#######################################"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282100314
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Running the Image Search for all test queries  \n",
        "**"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter    \n",
        "  \n",
        "def sort_by_frequency(lst):  \n",
        "   \n",
        "    freq_dict = Counter(lst)   \n",
        "    return sorted(lst, key=lambda x: (-freq_dict[x], x))  \n",
        "\n",
        "\n",
        "def sort_by_sum_scores(results):  \n",
        "   \n",
        "    sorted_ids = sorted(results.keys(), key=lambda x: -np.array(results[x]).sum())\n",
        "    return sorted_ids \n",
        "\n",
        "def sort_by_max_scores(results):  \n",
        "   \n",
        "    sorted_ids = sorted(results.keys(), key=lambda x: -np.array(results[x]).max())\n",
        "    return sorted_ids \n",
        "    \n",
        "def get_3D_results(slice_id_arr):\n",
        "    \"\"\"Baseline to get 3D img results from 2D slice results.\n",
        "    Uses 2D slice count of each 3D image in the result list.\n",
        "\n",
        "    input:\n",
        "     [\n",
        "        [slice_1_1, slice 1_2, ..], #results for query slice 1\n",
        "        [slice_2_1, slice 2_2, ..], #results for query slice 2\n",
        "        ...\n",
        "     ]\n",
        "\n",
        "     output:\n",
        "     [ img_id_1, img_id_2, .., img_id_n] # initial 3D images ranked by slice count\n",
        "\n",
        "    \"\"\"\n",
        "    results = []\n",
        "\n",
        "    for i, slice_ids in enumerate(slice_id_arr):\n",
        "        for slice_id in slice_ids:\n",
        "            slice_str = str(slice_id)\n",
        "            img_id = slice_str[:slice_str.rindex('_')]\n",
        "            results.append(img_id)\n",
        "\n",
        "    ## Select the aggregation method: max | sum | freq \n",
        "    #results = sort_by_max_scores(results)\n",
        "    #results = sort_by_sum_scores(results) \n",
        "    results = sort_by_frequency(results)\n",
        "    result_set = set()\n",
        "    final_results = []\n",
        "\n",
        "    for r in results:\n",
        "        if r not in result_set:\n",
        "            final_results.append(r)\n",
        "            result_set.add(r)\n",
        "    return final_results"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1700282100482
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "### SLICE-BASED SEARCH ###\n",
        "search_results1 = {'query_img': [], 'results': []}\n",
        "test_csv = pd.read_csv(test_set_csv)\n",
        "for img in test_csv['testing']: \n",
        "    query_img_id = img[:img.index('.')]\n",
        "    test_volume_emb = os.path.join(all_embs_folder, query_img_id + '.pkl')\n",
        "    search_results1['query_img'].append(query_img_id)\n",
        "    with open(test_volume_emb, \"rb\") as f:\n",
        "        embd = pickle.load(f)\n",
        "    D, I = index.retrieve_images(embd, 20) ## retrieving 20 slices for eval\n",
        "    results = get_3D_results(I)\n",
        "    search_results1['results'].append(' '.join(results))\n",
        "df1 = pd.DataFrame(search_results1)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282112395
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "### TEXT-BASED SEARCH ###\n",
        "search_results2 = {'query_img': [], 'results': []}\n",
        "test_csv = pd.read_csv(test_set_csv)\n",
        "captions_df = pd.read_csv(captionsFile) \n",
        "for img in test_csv['testing']: \n",
        "    query_img_id = img[:img.index('.')]\n",
        "    search_results2['query_img'].append(query_img_id)\n",
        "    caption=captions_df.loc[captions_df['filename']==query_img_id]['caption']\n",
        "    caption=str(caption).replace('\"', \"\")\n",
        "\n",
        "    D, I = ret_engine(caption, k=20) ## retrieving 20 slices for eval\n",
        "    results = get_3D_results(I)\n",
        "    search_results2['results'].append(' '.join(results))\n",
        "df2 = pd.DataFrame(search_results2)\n",
        "df2.to_csv(results_dir + '/' + method1 + '_results.csv', index=False)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1700282123353
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "aggregated_df = pd.read_csv(quantification_csv) \n",
        "aggregated_df['name'] = aggregated_df['name'].replace('.nii.gz', '')\n",
        "for i in range(len(aggregated_df['name'])):\n",
        "    filename = aggregated_df['name'][i]\n",
        "    aggregated_df['name'][i] = filename[:filename.index('.')] \n",
        "aggregated_df.head()"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282123549
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "##########################################################################################\n",
        "            # Evaluation of Caption-based Search \n",
        "##########################################################################################\n",
        "# Eval Top-k\n",
        "tumor_stage_eval  = {'query': [] } \n",
        "tumor_flag_eval = {'query': [] }\n",
        "\n",
        "for i in range(10):\n",
        "    tumor_stage_eval[f\"Top {i+1}\"] = []\n",
        "    tumor_flag_eval[f\"Top {i+1}\"] = []\n",
        "\n",
        "for query, res in zip(search_results2['query_img'], search_results2['results']):\n",
        "    query_metrics = aggregated_df.loc[aggregated_df['name'] == query].iloc[0]\n",
        "    \n",
        "    tumor_stage_eval['query'].append(query)\n",
        "    tumor_flag_eval['query'].append(query)\n",
        "\n",
        "    tumor_flag = organ + \"_cancer_flag\" \n",
        "    top_n = res.split(' ')\n",
        "    for k, top_k in enumerate(top_n):\n",
        "        if k > 9:\n",
        "            break\n",
        "\n",
        "        res_metrics = aggregated_df.loc[aggregated_df['name'] == top_k].iloc[0]\n",
        "        # tumor stage\n",
        "        tumorStage=0\n",
        "        if(query_metrics['cancer_stage']-res_metrics['cancer_stage'] == 0):\n",
        "            tumorStage=1 \n",
        "\n",
        "        tumor_stage_eval[f\"Top {k+1}\"].append(tumorStage)\n",
        "        tumor_flag_eval[f\"Top {k+1}\"].append(1-abs(query_metrics[tumor_flag]-res_metrics[tumor_flag]))\n",
        "        \n",
        "    if len(top_n) < 10:\n",
        "        for k in range(len(top_n), 10):\n",
        "            tumor_stage_eval[f\"Top {k+1}\"].append('NA')\n",
        "            tumor_flag_eval[f\"Top {k+1}\"].append('NA')\n",
        "\n",
        "tumor_stage_df = pd.DataFrame(tumor_stage_eval)\n",
        "tumor_stage_df.to_csv(results_dir + '/' + method1 + '_tumor_stage_eval.csv', index=False) \n",
        "tumor_flag_df = pd.DataFrame(tumor_flag_eval)\n",
        "tumor_flag_df.to_csv(results_dir + '/' + method1 + '_tumor_flag_eval.csv', index=False) \n",
        "\n",
        "##########################################################################################    \n",
        "print(\"### Results of Caption-based Search\")\n",
        "## Tumor Flag\n",
        "print(\"# Tumor Flag: \")\n",
        "tumor_flag_df['RowMean'] = tumor_flag_df.iloc[:, 1:4].mean(axis=1)\n",
        "overall_mean = tumor_flag_df['RowMean'].mean()\n",
        "print(\"tumor_flag_p@3: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "tumor_flag_df['RowMean'] = tumor_flag_df.iloc[:, 1:6].mean(axis=1)\n",
        "overall_mean = tumor_flag_df['RowMean'].mean()\n",
        "print(\"tumor_flag_p@5: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "tumor_flag_df['RowMean'] = tumor_flag_df.iloc[:, 1:].mean(axis=1)\n",
        "overall_mean = tumor_flag_df['RowMean'].mean()\n",
        "print(\"tumor_flag_p@10: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "## Tumor Stage\n",
        "print(\"# Tumor Stage: \")\n",
        "tumor_stage_df['RowMean'] = tumor_stage_df.iloc[:, 1:4].mean(axis=1)\n",
        "overall_mean = tumor_stage_df['RowMean'].mean()\n",
        "print(\"tumor_stage_p@3: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "tumor_stage_df['RowMean'] = tumor_stage_df.iloc[:, 1:6].mean(axis=1)\n",
        "overall_mean = tumor_stage_df['RowMean'].mean()\n",
        "print(\"tumor_stage_p@5: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "tumor_stage_df['RowMean'] = tumor_stage_df.iloc[:, 1:].mean(axis=1)\n",
        "overall_mean = tumor_stage_df['RowMean'].mean()\n",
        "print(\"tumor_stage_p@10: \", '{:.4f}'.format(overall_mean))"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282123720
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Method 2 - Average Precision"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ref_tumor_flag = []\n",
        "all_res_tumor_flag = []\n",
        "ref_tumor_stage = []\n",
        "all_res_tumor_stage = []\n",
        "\n",
        "search_results = search_results2\n",
        "\n",
        "for query, res in zip(search_results['query_img'], search_results['results']):\n",
        "    query_metrics = aggregated_df.loc[aggregated_df['name'] == query].iloc[0]\n",
        "    tumor_flag = organ + \"_cancer_flag\" \n",
        "    ref_tumor_flag.append(query_metrics[tumor_flag])\n",
        "    ref_tumor_stage.append(query_metrics['cancer_stage'])\n",
        "    top_n = res.split(' ')\n",
        "    res_tumor_flag = []\n",
        "    res_tumor_stage = []\n",
        "    for k, top_k in enumerate(top_n):\n",
        "        if k > 9:\n",
        "            break\n",
        "        res_metrics = aggregated_df.loc[aggregated_df['name'] == top_k].iloc[0]\n",
        "        res_tumor_flag.append(res_metrics[tumor_flag])\n",
        "        res_tumor_stage.append(res_metrics['cancer_stage'])\n",
        "    all_res_tumor_flag.append(res_tumor_flag)\n",
        "    all_res_tumor_stage.append(res_tumor_stage)\n",
        "\n",
        "\n",
        "## Adaptation for Average Precision (AP) computation\n",
        "## AP - Tumor Flag\n",
        "\n",
        "for i, q in enumerate(ref_tumor_flag):\n",
        "    if q == 0:\n",
        "        for j, res in enumerate(all_res_tumor_flag[i]):\n",
        "            if res == 0:\n",
        "                all_res_tumor_flag[i][j] = 1\n",
        "            else:\n",
        "                all_res_tumor_flag[i][j] = 0\n",
        "\n",
        "## AP - Tumor Stage\n",
        "\n",
        "for i, q in enumerate(ref_tumor_stage):\n",
        "    print(q)\n",
        "    for j, res in enumerate(all_res_tumor_stage[i]):\n",
        "        if res == q:\n",
        "            all_res_tumor_stage[i][j] = 1\n",
        "        else:\n",
        "            all_res_tumor_stage[i][j] = 0\n",
        "            "
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282123897
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#############\n",
        "## Tumor Flag\n",
        "ref_tumor_flag = np.array(ref_tumor_flag)\n",
        "all_res_tumor_flag = np.array(all_res_tumor_flag)\n",
        "\n",
        "ap = 0\n",
        "\n",
        "for i, q in enumerate(ref_tumor_flag):\n",
        "    \n",
        "    try:\n",
        "        flag_avg_precision =  average_precision_score(all_res_tumor_flag[i], list(range(len(all_res_tumor_flag[i])))[::-1])\n",
        "\n",
        "        if math.isnan(flag_avg_precision):\n",
        "            flag_avg_precision=0\n",
        "\n",
        "        ap = ap + flag_avg_precision\n",
        "    except:\n",
        "        print(all_res_tumor_flag[i])\n",
        "        \n",
        "\n",
        "print(\"flag_avg_precision: \", '{:.4f}'.format(ap/len(ref_tumor_flag)))\n"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282124078
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#############\n",
        "## Tumor Stage\n",
        "ref_tumor_stage = np.array(ref_tumor_stage)\n",
        "all_res_tumor_stage = np.array(all_res_tumor_stage)\n",
        "\n",
        "ap = 0\n",
        "\n",
        "for i, q in enumerate(ref_tumor_stage):\n",
        "    \n",
        "    try:\n",
        "        stage_avg_precision =  average_precision_score(all_res_tumor_stage[i], list(range(len(all_res_tumor_flag[i])))[::-1])\n",
        "        \n",
        "        if math.isnan(stage_avg_precision):\n",
        "            stage_avg_precision=0\n",
        "        \n",
        "        ap = ap + stage_avg_precision\n",
        "    except:\n",
        "        print(all_res_tumor_stage[i])\n",
        "\n",
        "print(\"stage_avg_precision: \", '{:.4f}'.format(ap/len(ref_tumor_stage)))\n",
        "\n"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282124265
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Hybrid Search (captions + slices)**"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## Combining the search results lists\n",
        "def interleave(result_list1, result_list2, k):\n",
        "    interleaved_results = []\n",
        "\n",
        "    for x, y in zip(result_list1, result_list2):\n",
        "        if x not in interleaved_results:\n",
        "            interleaved_results.append(x)\n",
        "        \n",
        "        if y not in interleaved_results:\n",
        "            interleaved_results.append(y)\n",
        "\n",
        "    if len(interleaved_results) > k:\n",
        "        interleaved_results = interleaved_results[:k]\n",
        "\n",
        "    return interleaved_results\n",
        "\n",
        "interleaved_results = {'query_img': [], 'results': []}\n",
        "\n",
        "i = 0\n",
        "for i, query_img in enumerate(search_results1['query_img']):\n",
        "    assert query_img == search_results2['query_img'][i]\n",
        "\n",
        "    results1 = search_results1['results'][i].split(' ')\n",
        "    results2 = search_results2['results'][i].split(' ')\n",
        "\n",
        "    ir = interleave(results1, results2, 10)\n",
        "\n",
        "    interleaved_results['query_img'].append(query_img)\n",
        "    interleaved_results['results'].append(' '.join(ir))\n",
        "    \n",
        "\n",
        "interleaved_df = pd.DataFrame(interleaved_results)\n",
        "interleaved_df.head()\n",
        "interleaved_df.to_csv(results_dir + '/' + method2 + '_results.csv', index=False)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282124431
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "##########################################################################################\n",
        "            # Evaluation of Hybrid Search (captions+slices)\n",
        "##########################################################################################\n",
        "# Eval Top-k\n",
        "tumor_stage_eval  = {'query': [] } \n",
        "tumor_flag_eval = {'query': [] }\n",
        "\n",
        "for i in range(10):\n",
        "    tumor_stage_eval[f\"Top {i+1}\"] = []\n",
        "    tumor_flag_eval[f\"Top {i+1}\"] = []\n",
        "\n",
        "for query, res in zip(interleaved_results['query_img'], interleaved_results['results']):\n",
        "    query_metrics = aggregated_df.loc[aggregated_df['name'] == query].iloc[0]\n",
        "    \n",
        "    tumor_stage_eval['query'].append(query)\n",
        "    tumor_flag_eval['query'].append(query)\n",
        "\n",
        "    tumor_flag = organ + \"_cancer_flag\" \n",
        "    top_n = res.split(' ')\n",
        "    for k, top_k in enumerate(top_n):\n",
        "        if k > 9:\n",
        "            break\n",
        "\n",
        "        res_metrics = aggregated_df.loc[aggregated_df['name'] == top_k].iloc[0]\n",
        "        # tumor stage\n",
        "        tumorStage=0\n",
        "        if(query_metrics['cancer_stage']-res_metrics['cancer_stage'] == 0):\n",
        "            tumorStage=1 \n",
        "\n",
        "        tumor_stage_eval[f\"Top {k+1}\"].append(tumorStage)\n",
        "        tumor_flag_eval[f\"Top {k+1}\"].append(1-abs(query_metrics[tumor_flag]-res_metrics[tumor_flag]))\n",
        "        \n",
        "    if len(top_n) < 10:\n",
        "        for k in range(len(top_n), 10):\n",
        "            tumor_stage_eval[f\"Top {k+1}\"].append('NA')\n",
        "            tumor_flag_eval[f\"Top {k+1}\"].append('NA')\n",
        "\n",
        "tumor_stage_df = pd.DataFrame(tumor_stage_eval)\n",
        "tumor_stage_df.to_csv(results_dir + '/' + method2 + '_tumor_stage_eval.csv', index=False) \n",
        "tumor_flag_df = pd.DataFrame(tumor_flag_eval)\n",
        "tumor_flag_df.to_csv(results_dir + '/' + method2 + '_tumor_flag_eval.csv', index=False) \n",
        "\n",
        "##########################################################################################\n",
        "print(\"### Results of Hybrid Search (Captions + slices)\")    \n",
        "## Tumor Flag\n",
        "print(\"# Tumor Flag: \")\n",
        "tumor_flag_df['RowMean'] = tumor_flag_df.iloc[:, 1:4].mean(axis=1)\n",
        "overall_mean = tumor_flag_df['RowMean'].mean()\n",
        "print(\"tumor_flag_p@3: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "tumor_flag_df['RowMean'] = tumor_flag_df.iloc[:, 1:6].mean(axis=1)\n",
        "overall_mean = tumor_flag_df['RowMean'].mean()\n",
        "print(\"tumor_flag_p@5: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "tumor_flag_df['RowMean'] = tumor_flag_df.iloc[:, 1:].mean(axis=1)\n",
        "overall_mean = tumor_flag_df['RowMean'].mean()\n",
        "print(\"tumor_flag_p@10: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "## Tumor Stage\n",
        "print(\"# Tumor Stage: \")\n",
        "tumor_stage_df['RowMean'] = tumor_stage_df.iloc[:, 1:4].mean(axis=1)\n",
        "overall_mean = tumor_stage_df['RowMean'].mean()\n",
        "print(\"tumor_stage_p@3: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "tumor_stage_df['RowMean'] = tumor_stage_df.iloc[:, 1:6].mean(axis=1)\n",
        "overall_mean = tumor_stage_df['RowMean'].mean()\n",
        "print(\"tumor_stage_p@5: \", '{:.4f}'.format(overall_mean))\n",
        "\n",
        "tumor_stage_df['RowMean'] = tumor_stage_df.iloc[:, 1:].mean(axis=1)\n",
        "overall_mean = tumor_stage_df['RowMean'].mean()\n",
        "print(\"tumor_stage_p@10: \", '{:.4f}'.format(overall_mean))"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1700282124605
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Hybrid - Average Precision"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ref_tumor_flag = []\n",
        "all_res_tumor_flag = []\n",
        "ref_tumor_stage = []\n",
        "all_res_tumor_stage = []\n",
        "\n",
        "search_results = interleaved_results\n",
        "\n",
        "for query, res in zip(search_results['query_img'], search_results['results']):\n",
        "    query_metrics = aggregated_df.loc[aggregated_df['name'] == query].iloc[0]\n",
        "    tumor_flag = organ + \"_cancer_flag\" \n",
        "    ref_tumor_flag.append(query_metrics[tumor_flag])\n",
        "    ref_tumor_stage.append(query_metrics['cancer_stage'])\n",
        "    top_n = res.split(' ')\n",
        "    res_tumor_flag = []\n",
        "    res_tumor_stage = []\n",
        "    for k, top_k in enumerate(top_n):\n",
        "        if k > 9:\n",
        "            break\n",
        "        res_metrics = aggregated_df.loc[aggregated_df['name'] == top_k].iloc[0]\n",
        "        res_tumor_flag.append(res_metrics[tumor_flag])\n",
        "        res_tumor_stage.append(res_metrics['cancer_stage'])\n",
        "    all_res_tumor_flag.append(res_tumor_flag)\n",
        "    all_res_tumor_stage.append(res_tumor_stage)\n",
        "\n",
        "\n",
        "## Adaptation for Average Precision (AP) computation\n",
        "## AP - Tumor Flag\n",
        "\n",
        "for i, q in enumerate(ref_tumor_flag):\n",
        "    if q == 0:\n",
        "        for j, res in enumerate(all_res_tumor_flag[i]):\n",
        "            if res == 0:\n",
        "                all_res_tumor_flag[i][j] = 1\n",
        "            else:\n",
        "                all_res_tumor_flag[i][j] = 0\n",
        "\n",
        "## AP - Tumor Stage\n",
        "\n",
        "for i, q in enumerate(ref_tumor_stage):\n",
        "    print(q)\n",
        "    for j, res in enumerate(all_res_tumor_stage[i]):\n",
        "        if res == q:\n",
        "            all_res_tumor_stage[i][j] = 1\n",
        "        else:\n",
        "            all_res_tumor_stage[i][j] = 0\n",
        "            "
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282124797
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#############\n",
        "## Tumor Flag\n",
        "ref_tumor_flag = np.array(ref_tumor_flag)\n",
        "all_res_tumor_flag = np.array(all_res_tumor_flag)\n",
        "\n",
        "ap = 0\n",
        "\n",
        "for i, q in enumerate(ref_tumor_flag):\n",
        "    \n",
        "    try:\n",
        "        flag_avg_precision =  average_precision_score(all_res_tumor_flag[i], list(range(len(all_res_tumor_flag[i])))[::-1])\n",
        "\n",
        "        if math.isnan(flag_avg_precision):\n",
        "            flag_avg_precision=0\n",
        "\n",
        "        ap = ap + flag_avg_precision\n",
        "    except:\n",
        "        print(all_res_tumor_flag[i])\n",
        "\n",
        "print(\"flag_avg_precision: \", '{:.4f}'.format(ap/len(ref_tumor_flag)))\n"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282124965
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#############\n",
        "## Tumor Stage\n",
        "ref_tumor_stage = np.array(ref_tumor_stage)\n",
        "all_res_tumor_stage = np.array(all_res_tumor_stage)\n",
        "\n",
        "ap = 0\n",
        "\n",
        "for i, q in enumerate(ref_tumor_stage):\n",
        "    \n",
        "    try:\n",
        "        stage_avg_precision =  average_precision_score(all_res_tumor_stage[i], list(range(len(all_res_tumor_flag[i])))[::-1])\n",
        "        \n",
        "        if math.isnan(stage_avg_precision):\n",
        "            stage_avg_precision=0\n",
        "        \n",
        "        ap = ap + stage_avg_precision\n",
        "    except:\n",
        "        print(all_res_tumor_stage[i])\n",
        "\n",
        "print(\"stage_avg_precision: \", '{:.4f}'.format(ap/len(ref_tumor_stage)))"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1700282125141
        }
      }
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "python38-azureml"
    },
    "kernelspec": {
      "name": "python38-azureml",
      "language": "python",
      "display_name": "Python 3.8 - AzureML"
    },
    "language_info": {
      "name": "python",
      "version": "3.8.5",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      },
      "ms_spell_check": {
        "ms_spell_check_language": "en"
      }
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}