multimodal_Text-Tag.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multimodal Text - Tags (MTT)\n",
    "\n",
    "&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;\n",
    "<img align=\"left\" src=\"Images\\text_tags.png\" width=\"400\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Utils import load_data, preprocessing, model_performances\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import KFold\n",
    "import json\n",
    "from sklearn.metrics import classification_report\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_column = \"misogynous\"\n",
    "input_columns = ['text_USE', 'tags_USE']\n",
    "threshold = 0.4\n",
    "\n",
    "embed_size = 512  # 512-length array with Universal Sentence Encoder algorithm\n",
    "input_shape = embed_size*2\n",
    "batch_size = 64\n",
    "epochs = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________load training data ___________________________________________________\n",
    "meme_df = load_data.load_clarifai_training_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "meme_df['text_USE'] = preprocessing.use_preprocessing(meme_df, 'Text Transcription')\n",
    "meme_df['tags_USE'] = preprocessing.meme_tag_embedding(meme_df, 'clarifai')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>misogynous</th>\n",
       "      <th>Text Transcription</th>\n",
       "      <th>clarifai</th>\n",
       "      <th>text_USE</th>\n",
       "      <th>tags_USE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5827.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>*Gets on Pornhub* Woman on Porn Ad: \"Are you s...</td>\n",
       "      <td>cartoon child man</td>\n",
       "      <td>[0.04223586246371269, -0.028678087517619133, 0...</td>\n",
       "      <td>[0.05812954778472582, -0.019499183321992557, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2454.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>When your high school girlfriend finally turns...</td>\n",
       "      <td>cartoon man woman</td>\n",
       "      <td>[-0.016387682408094406, -0.004176544025540352,...</td>\n",
       "      <td>[0.05792989581823349, -0.02607914184530576, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6492.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Me every time I refuse to objectify women I ca...</td>\n",
       "      <td>woman</td>\n",
       "      <td>[0.011072046123445034, -0.023999786004424095, ...</td>\n",
       "      <td>[0.04986587166786194, -0.04508301243185997, -0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2054.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Verizon Q Search News r/kotakuinaction2 There ...</td>\n",
       "      <td>cartoon man woman</td>\n",
       "      <td>[-0.02618846856057644, -0.034125760197639465, ...</td>\n",
       "      <td>[0.05792989581823349, -0.02607914184530576, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5388.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>me watching a horror movie NO DUDE WASN'T ME s...</td>\n",
       "      <td>animal cartoon</td>\n",
       "      <td>[-0.0059230634942650795, 0.006091502029448748,...</td>\n",
       "      <td>[0.060026247054338455, -0.003956574015319347, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  file_name  misogynous                                 Text Transcription  \\\n",
       "0  5827.jpg           0  *Gets on Pornhub* Woman on Porn Ad: \"Are you s...   \n",
       "1  2454.jpg           0  When your high school girlfriend finally turns...   \n",
       "2  6492.jpg           1  Me every time I refuse to objectify women I ca...   \n",
       "3  2054.jpg           1  Verizon Q Search News r/kotakuinaction2 There ...   \n",
       "4  5388.jpg           0  me watching a horror movie NO DUDE WASN'T ME s...   \n",
       "\n",
       "            clarifai                                           text_USE  \\\n",
       "0  cartoon child man  [0.04223586246371269, -0.028678087517619133, 0...   \n",
       "1  cartoon man woman  [-0.016387682408094406, -0.004176544025540352,...   \n",
       "2              woman  [0.011072046123445034, -0.023999786004424095, ...   \n",
       "3  cartoon man woman  [-0.02618846856057644, -0.034125760197639465, ...   \n",
       "4     animal cartoon  [-0.0059230634942650795, 0.006091502029448748,...   \n",
       "\n",
       "                                            tags_USE  \n",
       "0  [0.05812954778472582, -0.019499183321992557, 0...  \n",
       "1  [0.05792989581823349, -0.02607914184530576, 0....  \n",
       "2  [0.04986587166786194, -0.04508301243185997, -0...  \n",
       "3  [0.05792989581823349, -0.02607914184530576, 0....  \n",
       "4  [0.060026247054338455, -0.003956574015319347, ...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meme_df.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models on training data\n",
    "Training data are split in 10Fold and used to train models. Trained models are tested on the test fold. Models take as input text embedding and the mean-embed of clarifai tags.\n",
    "\n",
    "Predictions are saved in './Multimodal/predictions/text_tags_pred_10Fold.csv' file, with the following columns:\n",
    "    id: meme unique id\n",
    "    real: true label\n",
    "    pred: predicted value.\n",
    "\n",
    "Model performances are saved in './Multimodal/performances/text_tags_results_10Fold.txt' file.\n",
    "Models are not saved. To save models add command 'model.save(\"my_model\")' than load them with the command\n",
    "'model = keras.models.load_model(\"my_model\")'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________Utils ___________________________________________________\n",
    "#if not os.path.exists('./multimodal_text_tags/models'):\n",
    "#    os.makedirs('./multimodal_text_tags/models')\n",
    "\n",
    "if not os.path.exists('./Multimodal/predictions'):\n",
    "    os.makedirs('./Multimodal/predictions')\n",
    "\n",
    "if not os.path.exists('./Multimodal/performances'):\n",
    "    os.makedirs('./Multimodal/performances')\n",
    "\n",
    "#path_models = './multimodal_text_tags/models'\n",
    "file_out = './Multimodal/performances/text_tags_results_10Fold.txt'\n",
    "predictions_csv_path = './Multimodal/predictions/text_tags_pred_10Fold.csv'\n",
    "\n",
    "file = open(file_out, 'a+')\n",
    "file.truncate(0)  # erase file content\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________train model on training data 10Fold________________________________________\n",
    "kf = KFold(n_splits=10, shuffle=False)\n",
    "\n",
    "iteration = 0\n",
    "real_values = np.array([])\n",
    "predict_values = np.array([])\n",
    "ids = np.array([])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for train_index, test_index in kf.split(meme_df):  # split into train and test\n",
    "    preprocessing.set_seed(iteration)\n",
    "    x_train, y_train, x_val, y_val, x_test, y_test = preprocessing.elaborate_data_10fold(meme_df,\n",
    "                                                                                        train_index,\n",
    "                                                                                        test_index,\n",
    "                                                                                        iteration,\n",
    "                                                                                        input_columns,\n",
    "                                                                                        label_column)\n",
    "    model, history = model_performances.get_trained_model(x_train, \n",
    "                            y_train, \n",
    "                            x_val, \n",
    "                            y_val,\n",
    "                            input_shape=input_shape, \n",
    "                            activation_function='LeakyReLU', \n",
    "                            neurons=input_shape/2, \n",
    "                            dropout=0.2, \n",
    "                            epochs=100)\n",
    "    iteration = iteration + 1\n",
    "\n",
    "    # make prediction on training data\n",
    "    pred = model.predict(x_test, batch_size=batch_size)\n",
    "\n",
    "    predict_values = np.append(predict_values, pred)\n",
    "    real_values = np.append(real_values, y_test)\n",
    "    ids = np.append(ids, meme_df.iloc[test_index, :]['file_name'].tolist())\n",
    "\n",
    "    result_df = meme_df.iloc[test_index, [0, 1]]\n",
    "    result_df['score_col'] = pred\n",
    "\n",
    "    # write on file\n",
    "    file = open(file_out, \"a+\")\n",
    "    file.write('\\n\\nITERAZIONE ' + str(iteration) + '\\n')\n",
    "    file.write(json.dumps(model_performances.compute_confusion_rates(result_df, 'score_col', 'misogynous', threshold)))\n",
    "    file.write('\\n') \n",
    "    file.write(classification_report(result_df['misogynous'].values, (result_df['score_col']>threshold).astype(int).values, target_names=['not_mis','mis']))\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# results dataframe, save predictions\n",
    "result_df = pd.DataFrame({'id': ids, 'real': real_values.astype(int), 'pred': predict_values})\n",
    "result_df.to_csv(predictions_csv_path, index=False, sep='\\t')\n",
    "\n",
    "# Overall metrics _ write on file\n",
    "file = open(file_out, \"a+\")\n",
    "file.write('\\n\\n10 Fold Results ' + str(iteration) + '\\n')\n",
    "file.write(json.dumps(model_performances.compute_confusion_rates(result_df, 'pred', 'real', threshold)))\n",
    "file.write('\\n') \n",
    "file.write(classification_report(result_df['real'].values, (result_df['pred']>threshold).astype(int).values, target_names=['not_mis','mis']))\n",
    "file.write('\\n AUC:') \n",
    "file.write(str(model_performances.compute_auc(result_df['real'].values, result_df['pred'].values)))\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tpr': 0.87,\n",
       " 'tnr': 0.7398,\n",
       " 'fpr': 0.2602,\n",
       " 'fnr': 0.13,\n",
       " 'precision': 0.7697752610157494,\n",
       " 'recall': 0.87,\n",
       " 'accuracy': 0.8049,\n",
       " 'f1': 0.8168247112947142,\n",
       " 'auc': 0.8892745}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_performances.compute_confusion_rates(result_df, 'pred', 'real', threshold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Test and Synthetic Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Test and preprocessing\n",
    "test_df = load_data.load_clarifai_test_data()\n",
    "test_df['text_USE'] = preprocessing.use_preprocessing(test_df, 'Text Transcription')\n",
    "test_df['tags_USE'] = preprocessing.meme_tag_embedding(test_df, 'clarifai')\n",
    "\n",
    "x_test, y_test = preprocessing.elaborate_input(test_df, input_columns, label_column)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(140, 4)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# __________________________________ Load models and test on Syn dataset ________________________________________\n",
    "# Load synthetic data and compute USE embedding both for text and for Clarifai tags\n",
    "# Make prediction on synthetic with each model trained on training data\n",
    "# Create a csv with file_name, real misogyny value and a column for each model’s predictions\n",
    "\n",
    "syn_df = load_data.load_clarifai_syn_data()\n",
    "syn_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle according to the pre-established order\n",
    "syn_df = load_data.shuffle_syn(syn_df, 'file_name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>misogynous</th>\n",
       "      <th>Text Transcription</th>\n",
       "      <th>clarifai</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SYN_49.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>when your buddies makes fun of mcdonald employ...</td>\n",
       "      <td>cartoon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SIN_51.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>HAVE YOU SEEN MY SISTER? THIS IS A RECENT PHOT...</td>\n",
       "      <td>cartoon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SYN_69.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>KRAV MAGA The art of making your wrist sound l...</td>\n",
       "      <td>man woman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SYN_114.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Woman developer</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SYN_89.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Before foreplat vs. after foreplay DIY.DESPAIR...</td>\n",
       "      <td>animal cartoon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name  misogynous                                 Text Transcription  \\\n",
       "0   SYN_49.jpg           0  when your buddies makes fun of mcdonald employ...   \n",
       "1   SIN_51.jpg           0  HAVE YOU SEEN MY SISTER? THIS IS A RECENT PHOT...   \n",
       "2   SYN_69.jpg           0  KRAV MAGA The art of making your wrist sound l...   \n",
       "3  SYN_114.jpg           1                                    Woman developer   \n",
       "4   SYN_89.jpg           1  Before foreplat vs. after foreplay DIY.DESPAIR...   \n",
       "\n",
       "         clarifai  \n",
       "0         cartoon  \n",
       "1         cartoon  \n",
       "2       man woman  \n",
       "3          animal  \n",
       "4  animal cartoon  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syn_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "syn_df['text_USE'] = preprocessing.use_preprocessing(syn_df, 'Text Transcription')\n",
    "syn_df['tags_USE'] = preprocessing.meme_tag_embedding(syn_df, 'clarifai')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_test = test_df[['file_name', 'misogynous']].copy()\n",
    "res_syn = syn_df[['file_name', 'misogynous']].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models on Training-Test Data\n",
    "\n",
    "### Train models on training data with a 10Fold approach\n",
    " Load training data, compute with USE embedding for text and for categories.\n",
    " Fit 10 models using whole training data, each with a different fold as validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_syn, y_syn = preprocessing.elaborate_input(syn_df, input_columns, label_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________Utils ___________________________________________________\n",
    "MODELNAMES = ['multimodal_text_tags_v{}'.format(i) for i in range(10)]\n",
    "path_models = './Multimodal/models/Bias/'\n",
    "model_name = 'multimodal_text_tags'\n",
    "\n",
    "if not os.path.exists(path_models):\n",
    "    os.makedirs(path_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________train model on training data 10Fold________________________________________\n",
    "kf = KFold(n_splits=10, shuffle=False)\n",
    "iteration = 0\n",
    "\n",
    "for train_index, val_index in kf.split(meme_df):  # split into train and test\n",
    "    preprocessing.set_seed(iteration)\n",
    "    MODELNAME = MODELNAMES[iteration]\n",
    "    \n",
    "    x_train, y_train = preprocessing.elaborate_input(meme_df.iloc[train_index, :], input_columns, label_column)\n",
    "    x_val, y_val = preprocessing.elaborate_input(meme_df.iloc[val_index, :], input_columns, label_column)\n",
    "\n",
    "    model, history = model_performances.get_trained_model(x_train, \n",
    "                            y_train, \n",
    "                            x_val, \n",
    "                            y_val,\n",
    "                            input_shape=input_shape, \n",
    "                            activation_function='LeakyReLU', \n",
    "                            neurons=input_shape/2, \n",
    "                            dropout=0.2, \n",
    "                            epochs=epochs)\n",
    "\n",
    "    # save each model once for all\n",
    "    model.save(path_models + MODELNAME)\n",
    "    iteration = iteration + 1\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load models and test on Test dataset\n",
    "Load models trained on Training data with a 10-fold approach\n",
    "Load test data and compute USE embedding both for text and for clarifai tags Make prediction on test set with each model trained on training data.\n",
    "Create a csv with file_name, real misogyny value and a column for each model’s predictions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________Utils ___________________________________________________\n",
    "MODELNAMES = ['multimodal_text_tags_v{}'.format(i) for i in range(10)]\n",
    "path_models = './Multimodal/models/Bias/'\n",
    "path_results_test = './Multimodal/Predictions/predictions_test/'\n",
    "path_results_syn = './Multimodal/Predictions/predictions_syn/'\n",
    "model_name = 'multimodal_text_tags'\n",
    "path_performances = './Multimodal/Performances/'\n",
    "\n",
    "file_out_test = \"./Multimodal/Performances/text_tags_Results_Test.txt\"\n",
    "file_out_syn = \"./Multimodal/Performances/text_tags_Results_Syn.txt\"\n",
    "file_out_bias = \"./Multimodal/Performances/text_tags_Results_Bias.txt\"\n",
    "\n",
    "for path in [path_results_test, path_results_syn, path_performances]:\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "\n",
    "for file_name in [file_out_test, file_out_syn, file_out_bias]:\n",
    "    file = open(file_name, 'a+')\n",
    "    file.truncate(0)  # erase file content\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ______________________________ retrieve saved 10 models and make predictions _________________________________\n",
    "kf = KFold(n_splits=10, shuffle=False)\n",
    "syn_folds = kf.split(syn_df)\n",
    "pred_syn=[]\n",
    "syn_10_df=pd.DataFrame()\n",
    "\n",
    "for MODELNAME in MODELNAMES:\n",
    "    # LOAD MODEL\n",
    "    loaded_model = keras.models.load_model(path_models + MODELNAME)\n",
    "\n",
    "    # make prediction on test\n",
    "    predict_values = loaded_model.predict(x_test, batch_size=batch_size)\n",
    "    res_test[MODELNAME] = pd.DataFrame(predict_values)[0]\n",
    "\n",
    "    # make prediction on syn\n",
    "    predict_values = loaded_model.predict(x_syn, batch_size=batch_size)\n",
    "    res_syn[MODELNAME] = pd.DataFrame(predict_values)[0]\n",
    "\n",
    "    # performances on splitted Syn\n",
    "    _, test_syn = next(syn_folds)\n",
    "    syn_10_df['label_'+MODELNAME]=list(res_syn[label_column][test_syn].values)\n",
    "    syn_10_df[MODELNAME]= list(res_syn[MODELNAME][test_syn].values)\n",
    "    syn_10_df['file_name_'+MODELNAME]=list(res_syn['file_name'][test_syn].values)\n",
    "\n",
    "res_test.to_csv(path_results_test + \"baseline_\" + model_name + \"_scores.tsv\", sep=\"\\t\", index=False)\n",
    "res_syn.to_csv(path_results_syn + \"baseline_\" + model_name + \"_SYN_scores.tsv\", sep=\"\\t\", index=False)\n",
    "\n",
    "model_performances.plot_model_family_auc(res_test, MODELNAMES, label_column)\n",
    "\n",
    "model_performances.confusion_rates_on_file(file_out_test, res_test, MODELNAMES, label_column, threshold)\n",
    "model_performances.confusion_rates_on_file(file_out_syn, res_syn, MODELNAMES, label_column, threshold)\n",
    "model_performances.confusion_rates_on_file_10Fold_syn(file_out_syn, syn_10_df, MODELNAMES, threshold)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# __________________________________ Identity Elements __________________________________\n",
    "# Load Identity Terms and Identity Tags\n",
    "# NB: Only the identity terms and tags present at least in one misogynous and one non-misogynous meme are considered\n",
    "\n",
    "Identity_Terms = load_data.read_clear_identity_terms()\n",
    "Identity_Tags = load_data.read_clear_identity_tags()\n",
    "\n",
    "res_syn = res_syn.merge(load_data.load_syn_identity_data().drop(columns=['misogynous', 'Text Transcription']),\n",
    "                        how='inner', on='file_name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# _________________________________Compute Bias Metrics_____________________________________________________\n",
    "# Computes per-subgroup metrics for all subgroups and a list of models.\n",
    "subgroups = Identity_Terms\n",
    "model_performances.compute_bias_metrics_for_models(res_syn,\n",
    "                                                   subgroups,\n",
    "                                                   MODELNAMES,\n",
    "                                                   label_column)\n",
    "\n",
    "model_performances.bias_metrics_on_file(file_out_bias, res_test, res_syn, Identity_Terms, MODELNAMES, label_column)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# _________________________________Compute Bias Metrics Multilabel_____________________________________________________\n",
    "\n",
    "model_performances.multilabel_bias_metrics_on_file(file_out_bias, res_test, res_syn, Identity_Terms, Identity_Tags,\n",
    "                                                   MODELNAMES, label_column)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('ProjEnv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13 (default, Mar 28 2022, 06:59:08) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "036d5e79b2f89acf0b2080b6b489e4897d66fffe68690462bc44e5a56924deac"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}