load_capstone_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a51a2955-0e4e-45f7-8b46-fb44b7866294",
   "metadata": {},
   "source": [
    "### Author: `Winston Menzies`\n",
    "> #### Imperial College Business School\n",
    "> #### Professional Certificate in Machine Learning and Artifical Intelligence\n",
    "#### Date: `18 January 2024`\n",
    "#### Student ID: `484 (Class of 23-09)`\n",
    "#### Function: `Load Capstone Results`\n",
    "#### Usage: \\<TODO\\>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "362452c1-885e-44d2-9a23-42e136f8a406",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, time\n",
    "import pathlib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.lines import Line2D\n",
    "import copy\n",
    "from tabulate import tabulate\n",
    "from datetime import datetime\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore', category=DeprecationWarning)\n",
    "np.set_printoptions(threshold=np.inf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e79178e-23d0-4aa9-a74f-a637ec3d6eb4",
   "metadata": {},
   "source": [
    "#### - declare constant values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dacef216-a12a-419b-827c-1041c0f36706",
   "metadata": {},
   "outputs": [],
   "source": [
    "INITIAL_DATA_PATH = './initial_data/'\n",
    "INITIAL_DATA1_PATH = './initial_data1/'\n",
    "INITIAL_DATA2_PATH = './initial_data2/'\n",
    "INITIAL_FILENAME_INPUTS = 'initial_inputs.npy'\n",
    "INITIAL_FILENAME_OUTPUTS = 'initial_outputs.npy'\n",
    "IMAGE_DIRECTORY = './docs/images/'\n",
    "FILENAME_INPUTS = 'inputs.npy'\n",
    "FILENAME_OUTPUTS = 'outputs.npy'\n",
    "UPDATE_DATA_PATH = '/update_data/'\n",
    "FUNCTION_DIRECTORY = 'function_'\n",
    "DATA_PATH = './data/'\n",
    "DATA_DIR_PREFIX = 'week_'\n",
    "DATA_STUDENT_ID_SEPARATOR1 = '-'\n",
    "DATA_STUDENT_ID_SEPARATOR2 = '_'\n",
    "DATA_TYPE_OBSERVATIONS = 'observations'\n",
    "DATA_TYPE_QUERIES = 'queries'\n",
    "DATA_FILENAME = 'capstone_results.csv'\n",
    "LARGEST_FUNCTION_DIMENSION = 8\n",
    "TOTAL_FUNCTIONS = 8\n",
    "INDEX_STUDENT_FILE = 1\n",
    "INDEX_WEEK_NUM = 0\n",
    "INDEX_STUDENT_ID = 0\n",
    "INDEX_DATA_TYPE = 1\n",
    "INDEX_DATA_FILENAME = 2\n",
    "INDEX_FUNCTION_NUMBER = 2\n",
    "INDEX_FUNCTION_VALUE = 3\n",
    "INDEX_INPUT_VALUE = 2\n",
    "INDEX_OUTPUT_VALUE = TOTAL_FUNCTIONS + INDEX_FUNCTION_VALUE\n",
    "TOTAL_COLUMNS = INDEX_OUTPUT_VALUE + 1\n",
    "BLANK_VALUE = None\n",
    "BLANK_SPACE = ' '*6\n",
    "TAB_SPACE = ' '*4\n",
    "COLOR_RED = '\\033[1;31m'\n",
    "COLOR_BLUE = '\\033[1;34m'\n",
    "RESET = '\\033[0m'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b1a2426-41ba-43ab-943a-3ec60dbdd010",
   "metadata": {},
   "source": [
    "#### - define `class` to encapsulate and manage the loading process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6481852-7ec5-48d0-94ea-db3d4b5cc0a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CapstoneResultLoader(object):\n",
    "    \"\"\"\n",
    "    Load Capstone Results\n",
    "    Collate students' queries and observations\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        self._file_list = [] #store result files\n",
    "        self._data_observations = [] #store observations\n",
    "        self._data_queries = [] #store queries\n",
    "        self._data_results = [] #store queries + observations \n",
    "        self._data_header = [\"Week\", \"Student\", \"Fn\", \n",
    "                             \"X1\", \"X2\", \"X3\", \"X4\", \"X5\", \"X6\", \"X7\", \"X8\", \n",
    "                             \"Y\"]\n",
    "        self._initial_data = {} #store merge initial input and output values\n",
    "        self._initial_data1 = {} #store initial input and output values\n",
    "        self._initial_data2 = {} #store initial input and output values\n",
    "        self._update_data = {} #store appended input and output values\n",
    "        self.initialisation()\n",
    "        \n",
    "    def initialisation(self):\n",
    "        self._load_data()\n",
    "        self._load_data_result()\n",
    "        self._format_data_result()\n",
    "        self._load_initial_data1()\n",
    "        self._load_initial_data2()\n",
    "        self._merge_initial_data1_and_data2()\n",
    "        self.print_file_list()\n",
    "        self.update_initial_function_data()\n",
    "\n",
    "    def get_data_header(self):\n",
    "        return self._data_header\n",
    "        \n",
    "    def get_data_observations(self):\n",
    "        return self._data_observations\n",
    "        \n",
    "    def get_data_queries(self):\n",
    "        return self._data_queries\n",
    "        \n",
    "    def get_data_results(self):\n",
    "        return self._data_results\n",
    "        \n",
    "    def get_data_size(self):\n",
    "        dim = self._data_results.shape\n",
    "        print(f'Rows = {COLOR_BLUE}{dim[0]}{RESET} Cols = {COLOR_BLUE}{dim[1]}{RESET}')\n",
    "        \n",
    "    def get_file_list(self):\n",
    "        return self._file_list\n",
    "        \n",
    "    def get_initial_function_data(self):\n",
    "        return self._initial_data\n",
    "\n",
    "    def get_initial_function_data1(self):\n",
    "        return self._initial_data1\n",
    "\n",
    "    def get_initial_function_data2(self):\n",
    "        return self._initial_data2\n",
    "\n",
    "    def get_student_list(self):\n",
    "        list = []\n",
    "        for i in range(len(self._data_queries)):\n",
    "            student = self._data_queries[i][INDEX_STUDENT_FILE]\n",
    "            if list.count(student) == 0:\n",
    "                list.append(student)\n",
    "        return list\n",
    "        \n",
    "    def get_update_function_data(self):\n",
    "        return self._update_data\n",
    "\n",
    "    def get_week_list(self):\n",
    "        list = []\n",
    "        for i in range(len(self._data_queries)):\n",
    "            week = self._data_queries[i][INDEX_WEEK_NUM]\n",
    "            if list.count(week) == 0:\n",
    "                list.append(week)\n",
    "        return list\n",
    "        \n",
    "    def convert_to_pandas(self):\n",
    "        df = pd.DataFrame(self._data_results, columns=self._data_header)\n",
    "        return df\n",
    "        \n",
    "    def plot_observations(self, student_id = None, function_number = None):\n",
    "        if function_number is None:\n",
    "            fig, ax = plt.subplots(TOTAL_FUNCTIONS, 1, figsize=(8, 6 * TOTAL_FUNCTIONS))\n",
    "            for i in range(1, TOTAL_FUNCTIONS + 1):\n",
    "                self._plot_update_observations(i, ax[i - 1], student_id)\n",
    "        else:\n",
    "            fig, ax = plt.subplots(1, 1, figsize=(8, 6))\n",
    "            self._plot_update_observations(function_number, ax, student_id)\n",
    "        plt.show()\n",
    "        \n",
    "    def print_file_list(self):\n",
    "        self._format_file_list()\n",
    "        \n",
    "    def print_function_results(self, num):\n",
    "        list = []\n",
    "        for i in range(len(self._data_results)):\n",
    "            result = self._data_results[i]\n",
    "            if result[INDEX_FUNCTION_NUMBER] == num:\n",
    "                list.append(self._data_results[i])\n",
    "        self._print_table_results(list)\n",
    "        \n",
    "    def print_function_sample_total(self):\n",
    "        self._format_function_sample_total()\n",
    "        \n",
    "    def print_html_updated_data(self):\n",
    "        return self._format_html_updated_data(self._update_data)\n",
    "\n",
    "    def print_initial_function_data1(self):\n",
    "        return self._format_initial_function_data(self._initial_data1)\n",
    "\n",
    "    def print_initial_function_data2(self):\n",
    "        return self._format_initial_function_data(self._initial_data2)\n",
    "\n",
    "    def print_results(self, sort=False):\n",
    "        self._print_table_results(self._data_results, sort)\n",
    "        \n",
    "    def print_student_results(self, num, sort=False):\n",
    "        list = []\n",
    "        for i in range(len(self._data_results)):\n",
    "            result = self._data_results[i]\n",
    "            if result[INDEX_STUDENT_FILE] == num:\n",
    "                list.append(self._data_results[i])\n",
    "        self._print_table_results(list, sort)\n",
    "        \n",
    "    def print_updated_data(self):\n",
    "        return self._format_initial_function_data(self._update_data)\n",
    "\n",
    "    def save_plot_observations(self):\n",
    "        fig, ax = plt.subplots(TOTAL_FUNCTIONS, 1, figsize=(8, 6 * TOTAL_FUNCTIONS))\n",
    "        for i in range(1, TOTAL_FUNCTIONS + 1):\n",
    "            self._plot_update_observations(i, ax[i - 1], None)\n",
    "        # plt.gcf().set_size_inches(8, 48)\n",
    "        plt.savefig(self._formulate_filename('graph'), bbox_inches='tight')\n",
    "        plt.show()\n",
    "        plt.close()\n",
    "        \n",
    "    def save_results(self):\n",
    "        df = self.convert_to_pandas()\n",
    "        df.to_csv(self._formulate_filename('results'), index=False)\n",
    "        print(f'The capstone results file {COLOR_BLUE}{DATA_FILENAME}{RESET} has successfully saved.') \n",
    "        \n",
    "    def update_initial_function_data(self):\n",
    "        self._append_function_results()\n",
    "        self._save_updated_results(INITIAL_DATA_PATH, self._initial_data, 'initial_', 'Saving (merged) initial data files...')\n",
    "        self._save_updated_results(UPDATE_DATA_PATH, self._update_data, 'update_', 'Saving updated files...')\n",
    "\n",
    "    def _get_directory_folder(self, dir):\n",
    "        list = []\n",
    "        for path in pathlib.Path(dir).iterdir():\n",
    "            if path.is_dir():\n",
    "                pos = path.name.find(DATA_STUDENT_ID_SEPARATOR2)\n",
    "                if pos == -1:\n",
    "                   data_dir = path.name\n",
    "                else:\n",
    "                    data_dir = path.name[pos+1:pos+3]\n",
    "                list.append([data_dir, path])\n",
    "        # return sorted folders (by week no.) \n",
    "        return sorted(list, key=lambda x: x[0]) \n",
    "\n",
    "    def _get_function_results(self, num):\n",
    "        inputs = []\n",
    "        outputs = []\n",
    "        for i in range(len(self._data_results)):\n",
    "            result = self._data_results[i]\n",
    "            if result[INDEX_FUNCTION_NUMBER] == num:\n",
    "                values = []\n",
    "                # get input values for function \n",
    "                for i in range(INDEX_FUNCTION_VALUE, INDEX_FUNCTION_VALUE + LARGEST_FUNCTION_DIMENSION):\n",
    "                    if result[i] != BLANK_VALUE:\n",
    "                        values.append(result[i])\n",
    "                # append input values \n",
    "                inputs.append(values)\n",
    "                # get output value for function and append as list \n",
    "                outputs.append(result[INDEX_OUTPUT_VALUE])\n",
    "        return (np.array(inputs), np.array(outputs))\n",
    "\n",
    "    def _get_function_initial_observations(self, function_number):\n",
    "        if not self._initial_data:\n",
    "            return None\n",
    "        return self._initial_data[str(function_number)]['outputs']\n",
    "        \n",
    "    def _get_function_update_observations(self, function_number):\n",
    "        if not self._update_data:\n",
    "            return None\n",
    "        return self._update_data[str(function_number)]['outputs']\n",
    "        \n",
    "    def _get_observation_week_number(self, function_number, observation):\n",
    "        week_num = 0\n",
    "        a = self._data_results\n",
    "        result = a[np.where((a[:,INDEX_FUNCTION_NUMBER] == function_number) * \n",
    "                            (a[:,INDEX_OUTPUT_VALUE] == observation))]\n",
    "        if len(result) > 0:\n",
    "            return result[0][INDEX_WEEK_NUM]\n",
    "        return week_num\n",
    "        \n",
    "    def _get_week_number_color(self, week_number):\n",
    "        color = 'black'\n",
    "        if week_number == 1:\n",
    "            color = 'yellow'\n",
    "        elif week_number == 2:\n",
    "            color = 'lime'\n",
    "        elif week_number == 3:\n",
    "            color = 'dodgerblue'\n",
    "        elif week_number == 4:\n",
    "            color = 'beige'\n",
    "        elif week_number == 5:\n",
    "            color = 'cyan'\n",
    "        elif week_number == 6:\n",
    "            color = 'peru'\n",
    "        elif week_number == 7:\n",
    "            color = 'slategray'\n",
    "        else:\n",
    "            color = 'dodgerblue'\n",
    "        return color\n",
    "        \n",
    "    def _append_function_results(self):\n",
    "        self._update_data = {}\n",
    "        for key, value in self._initial_data.items():\n",
    "            inputs, outputs = self._get_function_results(int(key))\n",
    "            self._update_data[key] = dict({\n",
    "                \"inputs\" : np.vstack([value[\"inputs\"], inputs]),\n",
    "                \"outputs\" : np.hstack([value[\"outputs\"], outputs])\n",
    "            });\n",
    "\n",
    "    def _check_student_observation(self, function_number, student_id, observation):\n",
    "        if student_id is not None:\n",
    "            a = self._data_results\n",
    "            result = a[np.where((a[:,INDEX_FUNCTION_NUMBER] == function_number) * \n",
    "                                (a[:,INDEX_STUDENT_FILE] == student_id) * \n",
    "                                (a[:,INDEX_OUTPUT_VALUE] == observation))]\n",
    "            if len(result) > 0:\n",
    "                return True\n",
    "        return False\n",
    "        \n",
    "    def _check_updated_data_loaded(self, dir):\n",
    "        if not os.path.exists(dir):\n",
    "            os.makedirs(dir)\n",
    "        return dir\n",
    "        \n",
    "    def _create_updated_directory(self, dir):\n",
    "        if not os.path.exists(dir):\n",
    "            os.makedirs(dir)\n",
    "        return dir\n",
    "        \n",
    "    def _formulate_filename(self, type):\n",
    "        filename = \"\"\n",
    "        if type == 'results':\n",
    "            filename = DATA_FILENAME\n",
    "        elif type == 'graph':\n",
    "            now = datetime.now()\n",
    "            filename = f'{IMAGE_DIRECTORY}graph_{datetime.now().strftime(\"%y%m%d_%H%M\")}.jpg'\n",
    "        return filename\n",
    "\n",
    "    def _load_data(self):\n",
    "        dirs = self._get_directory_folder(DATA_PATH)\n",
    "        for dir in dirs:\n",
    "            data_file = self._process_directory(dir[1])\n",
    "            self._file_list.append([dir[0], data_file])\n",
    "                \n",
    "    def _load_data_observation(self, filename):\n",
    "        f = open(filename, 'r')\n",
    "        observations = []\n",
    "        values = \"\"\n",
    "        count = 0\n",
    "        for line in f.readlines():\n",
    "            observations.append(line.replace('\\n', '').strip())\n",
    "        f.close()\n",
    "        # process last set of observations only\n",
    "        # print(f'::_load_data_observation::{filename}\\nlength = [{len(observations)}]\\n{observations}\\n')\n",
    "        observations = observations[len(observations)-1]#[1:-1]\n",
    "        data = np.fromstring(observations[1:-1], dtype=float, sep=',')\n",
    "        return data\n",
    "    \n",
    "    def _load_data_query(self, filename):\n",
    "        f = open(filename, 'r')\n",
    "        queries = []\n",
    "        values = \"\"\n",
    "        count = 0\n",
    "        for line in f.readlines():\n",
    "            count +=1\n",
    "            if count == 1:\n",
    "                values = line.replace('\\n', '').strip() #store first line of query values\n",
    "            else:\n",
    "                #check if new (first line) set of query values\n",
    "                pos = line.find('[array([')\n",
    "                if pos == -1:\n",
    "                    values += line.replace('\\n', '').strip() #add query values\n",
    "                    # print(f'::_load_data_query::{filename} {count}\\n{values}')\n",
    "                else:\n",
    "                    #save current set of query values\n",
    "                    # print(f'::_load_data_query::{filename} {count}\\n{values}')\n",
    "                    queries.append(values)\n",
    "                    #store new (first line) set of query values\n",
    "                    values = line.replace('\\n', '').strip() \n",
    "                    count = 1\n",
    "        f.close()\n",
    "        # process last set of queries only\n",
    "        queries.append(values)\n",
    "        values = queries[len(queries)-1][1:-1]\n",
    "        # print(f'::_load_data_query::{filename}\\n{values}\\n')\n",
    "        values = values.split(\"),\")\n",
    "        data = []\n",
    "        for i in range(len(values)):\n",
    "            open_bracket_pos = values[i].find('[')\n",
    "            end_bracket_pos = values[i].find(']')\n",
    "            query = values[i][open_bracket_pos+1:end_bracket_pos]\n",
    "            data.append(np.fromstring(query, dtype=float, sep=','))\n",
    "            # print(f'::_load_data_query::{filename}\\n{i} {data}')\n",
    "        return data\n",
    "    \n",
    "    def _load_data_result(self):\n",
    "        query_len = 0\n",
    "        for i in range(len(self._file_list)):\n",
    "            for j in range(len(self._file_list[i][INDEX_STUDENT_FILE])):\n",
    "                student_file = self._file_list[i][INDEX_STUDENT_FILE][j]\n",
    "                if student_file[INDEX_DATA_TYPE] == DATA_TYPE_OBSERVATIONS:\n",
    "                    self._data_observations.append(self._load_data_observation(student_file[INDEX_DATA_FILENAME]))\n",
    "                else:\n",
    "                    queries = self._load_data_query(student_file[INDEX_DATA_FILENAME])\n",
    "                    if len(queries) > query_len:\n",
    "                        query_len = len(queries)\n",
    "                    self._data_queries.append([self._file_list[i][INDEX_WEEK_NUM], \n",
    "                                               student_file[INDEX_STUDENT_ID], \n",
    "                                               queries])\n",
    "\n",
    "    def _load_initial_data1(self):\n",
    "        for path in pathlib.Path(INITIAL_DATA1_PATH).iterdir():\n",
    "            if path.is_dir():\n",
    "                data_dir = path.name\n",
    "                # set key to directory number (last character)\n",
    "                self._initial_data1[data_dir[-1]] = dict({\n",
    "                    \"inputs\" : np.load(INITIAL_DATA1_PATH + '/' + data_dir + '/' + INITIAL_FILENAME_INPUTS),\n",
    "                    \"outputs\" : np.load(INITIAL_DATA1_PATH + '/' + data_dir + '/' + INITIAL_FILENAME_OUTPUTS)\n",
    "                });\n",
    "        \n",
    "    def _load_initial_data2(self):\n",
    "        for path in pathlib.Path(INITIAL_DATA2_PATH).iterdir():\n",
    "            if path.is_dir():\n",
    "                data_dir = path.name\n",
    "                # set key to directory number (last character)\n",
    "                self._initial_data2[data_dir[-1]] = dict({\n",
    "                    \"inputs\" : np.load(INITIAL_DATA2_PATH + '/' + data_dir + '/' + INITIAL_FILENAME_INPUTS),\n",
    "                    \"outputs\" : np.load(INITIAL_DATA2_PATH + '/' + data_dir + '/' + INITIAL_FILENAME_OUTPUTS)\n",
    "                });\n",
    "        \n",
    "    def _merge_initial_data1_and_data2(self):\n",
    "        self._initial_data = {}\n",
    "        for key, value in self._initial_data1.items():\n",
    "            inputs, outputs = self._get_function_results(int(key))\n",
    "            self._initial_data[key] = dict({\n",
    "                \"inputs\" : np.vstack([value[\"inputs\"], self._initial_data2[key][\"inputs\"]]),\n",
    "                \"outputs\" : np.hstack([value[\"outputs\"], self._initial_data2[key][\"outputs\"]])\n",
    "            });\n",
    "\n",
    "        \n",
    "    def _plot_update_observations(self, function_number, axis, student_id):\n",
    "        initial_observations = self._get_function_initial_observations(function_number)\n",
    "        update_observations = self._get_function_update_observations(function_number)\n",
    "        total_initial_observations = len(initial_observations)\n",
    "        total_observations = len(update_observations)\n",
    "        total_student_id_points = 0\n",
    "        initial_point_color = 'black'\n",
    "        student_point_color = 'lime'\n",
    "        student_id_color = 'fuchsia'\n",
    "        student_id_points = []\n",
    "        week_points = []\n",
    "        colors = []\n",
    "        point_size = 100\n",
    "        week_list = self.get_week_list()\n",
    "        for i in range(len(week_list)):\n",
    "            week_points.append(0)\n",
    "        # loop through all observation points\n",
    "        for x in range(1, total_observations+1):\n",
    "            y = update_observations[x-1]\n",
    "            week_number = int(self._get_observation_week_number(function_number, y))\n",
    "            point_size = 100\n",
    "            if x-1 < total_initial_observations:\n",
    "                # initial observations\n",
    "                color = initial_point_color\n",
    "                point_size = 60\n",
    "            else:\n",
    "                # student observations\n",
    "                color = self._get_week_number_color(week_number)\n",
    "                # only one point per student\n",
    "                # check student point hasn't already been counted, for current week\n",
    "                if (week_points[week_number - 1] == 0 and\n",
    "                    self._check_student_observation(function_number, student_id, y)):\n",
    "                    week_points[week_number - 1] +=1\n",
    "                    color = student_id_color\n",
    "                    total_student_id_points +=1\n",
    "                    student_id_points.append((week_number, x, y))\n",
    "                    point_size = 110\n",
    "            colors.append(color)\n",
    "            axis.scatter(x, y, s=point_size, color=color, edgecolor='black')\n",
    "        # rescale y axis\n",
    "        vmin = np.min(update_observations)\n",
    "        vmax = np.max(update_observations)\n",
    "        y_ticks = axis.get_yticks()\n",
    "        lower_tick_value = y_ticks[0]\n",
    "        upper_tick_value = y_ticks[len(y_ticks) - 1]\n",
    "        tick_unit = y_ticks[1] - y_ticks[0]\n",
    "        x_limits = axis.get_xlim()\n",
    "        y_limits = axis.get_ylim()\n",
    "        axis.set_ylim(vmin - (tick_unit * 0.5), vmax + (tick_unit * 0.5))\n",
    "        axis.set_xlabel('Samples')\n",
    "        axis.set_ylabel('Observations')\n",
    "        axis.set_facecolor('b')\n",
    "        # Shrink current axis by 20%\n",
    "        box = axis.get_position()\n",
    "        axis.set_position([box.x0, box.y0, box.width * 0.8, box.height])\n",
    "        # label student id points\n",
    "        for i in range(len(student_id_points)):\n",
    "            week = student_id_points[i][0]\n",
    "            x = student_id_points[i][1]\n",
    "            y = student_id_points[i][2]\n",
    "            axis.annotate(f'wk{int(week)}', color='white', fontsize=8,\n",
    "                          xy=(x, y), xycoords='data', xytext=(-8.0, 7.0), \n",
    "                          textcoords='offset points')\n",
    "        # Put a legend to the right of the current axis\n",
    "        legend_elements = []\n",
    "        # add key\n",
    "        legend_elements.append(Line2D([0], [0], label=f'initial data ({total_initial_observations})',\n",
    "                               color='w', markeredgecolor='black',\n",
    "                               marker='o', markersize=10, markerfacecolor=initial_point_color))\n",
    "        # add metrics\n",
    "        # show min and max values for initial observations\n",
    "        vmin = np.min(initial_observations)\n",
    "        vmax = np.max(initial_observations)\n",
    "        legend_elements.append(Line2D([0], [0], label=f'min value: {vmin:.4f}', color='w'))\n",
    "        legend_elements.append(Line2D([0], [0], label=f'max value: {vmax:.4f}', color='w'))\n",
    "        if total_student_id_points > 0:\n",
    "            legend_elements.append(Line2D([0], [0], label=f'student {student_id}',\n",
    "                                   color='w', markeredgecolor='black',\n",
    "                                   marker='o', markersize=10, markerfacecolor=student_id_color))\n",
    "        legend_elements.append(Line2D([0], [0], \n",
    "                               label=f'student data ({total_observations - total_initial_observations})', color='w'))\n",
    "        # add student data key colors\n",
    "        for i in range(len(week_points)):\n",
    "            legend_elements.append(Line2D([0], [0], \n",
    "                                   label=f'student data wk{i+1}',\n",
    "                                   color='w', markeredgecolor='black', \n",
    "                                   marker='o', markersize=10, markerfacecolor=self._get_week_number_color(i+1)))\n",
    "        axis.legend(handles=legend_elements, loc='center left', bbox_to_anchor=(1.014, 0.55))\n",
    "        axis.set_title(f'Function {function_number}')\n",
    "        \n",
    "    def _print_table_results(self, list, sort=False):\n",
    "        table = []\n",
    "        if sort:\n",
    "            list = sorted(list, key = lambda x: (x[2], x[0]))\n",
    "        for i in range(len(list)):\n",
    "            result = list[i]\n",
    "            values = []\n",
    "            for j in range(len(result)):\n",
    "                if result[j] == BLANK_VALUE:\n",
    "                    values.append(BLANK_SPACE)\n",
    "                else:\n",
    "                    if j > INDEX_FUNCTION_NUMBER:\n",
    "                        values.append(f'{result[j]:9.4f}')\n",
    "                    else:\n",
    "                        values.append(result[j])\n",
    "            table.append(values)\n",
    "        print(tabulate(table, headers=self._data_header))\n",
    "        \n",
    "    def _process_directory(self, dir):\n",
    "        list = []\n",
    "        for file in pathlib.Path(dir).iterdir():\n",
    "            if file.is_file():\n",
    "                list.append(self._format_data_file(file))\n",
    "        # sort list of result files by student_id and data_type\n",
    "        return sorted(list, key=lambda x: (x[0], x[1]))\n",
    "\n",
    "    def _format_initial_function_data(self, data):\n",
    "        table = []\n",
    "        headers = [\"Fn\", \"Sample\", \n",
    "                   \"X1\", \"X2\", \"X3\", \"X4\", \"X5\", \"X6\", \"X7\", \"X8\", \n",
    "                   \"Y\"]\n",
    "        for key, value in data.items():\n",
    "            inputs = value[\"inputs\"]\n",
    "            outputs = value[\"outputs\"]\n",
    "            for i in range(len(inputs)): #process function inputs\n",
    "                values = []\n",
    "                #append function number\n",
    "                values.append(key)\n",
    "                #append sample number\n",
    "                values.append(i+1)\n",
    "                total = len(inputs[i])\n",
    "                for j in range(total): #process input values\n",
    "                    values.append(f'{inputs[i][j]:9.6f}')\n",
    "                #append blank values (if required)\n",
    "                for k in range(LARGEST_FUNCTION_DIMENSION - total):\n",
    "                    values.append(BLANK_SPACE) \n",
    "                #append function input value, output value\n",
    "                values.append(f'{outputs[i]}')\n",
    "                table.append(values)\n",
    "        print(tabulate(table, headers=headers))\n",
    "        \n",
    "    def _format_html_updated_data(self, data):\n",
    "        html = []\n",
    "        html.append('<table>\\n')\n",
    "        # create table heading\n",
    "        html.append(f'{TAB_SPACE*1}<thead>\\n')\n",
    "        html.append(f'{TAB_SPACE*2}<tr>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">Fn</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">Sample</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">X1</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">X2</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">X3</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">X4</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">X5</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">X6</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">X7</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">X8</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*3}<th scope=\"col\">Y</th>\\n')\n",
    "        html.append(f'{TAB_SPACE*2}</tr>\\n')\n",
    "        html.append(f'{TAB_SPACE*1}</thead>\\n')\n",
    "        html.append(f'{TAB_SPACE*1}<tbody>\\n')\n",
    "        for key, value in data.items():\n",
    "            inputs = value[\"inputs\"]\n",
    "            outputs = value[\"outputs\"]\n",
    "            for i in range(len(inputs)): #process function inputs\n",
    "                html.append(f'{TAB_SPACE*2}<tr>\\n')\n",
    "                #append function number\n",
    "                html.append(f'{TAB_SPACE*3}<th scope=\"row\">{key}</th>\\n')\n",
    "                #append sample number\n",
    "                html.append(f'{TAB_SPACE*3}<td>{i+1}</td>\\n')\n",
    "                total = len(inputs[i])\n",
    "                for j in range(total): #process input values\n",
    "                    html.append(f'{TAB_SPACE*3}<td>{inputs[i][j]:6.4f}</td>\\n')\n",
    "                #append blank values (if required)\n",
    "                for k in range(LARGEST_FUNCTION_DIMENSION - total):\n",
    "                    html.append(f'{TAB_SPACE*3}<td>&nbsp;</td>\\n')\n",
    "                #append function input value, output value\n",
    "                html.append(f'{TAB_SPACE*3}<td>{outputs[i]}</td>\\n')\n",
    "                html.append(f'{TAB_SPACE*2}</tr>\\n')\n",
    "        html.append(f'{TAB_SPACE*1}</tbody>\\n')\n",
    "        html.append('</table>')\n",
    "        print(*html)\n",
    "        \n",
    "    def _format_data_result(self):\n",
    "        self._data_results = np.full((len(self._data_queries) * TOTAL_FUNCTIONS, TOTAL_COLUMNS),\n",
    "                                     fill_value = BLANK_VALUE)\n",
    "        for i in range(len(self._data_queries)):\n",
    "            queries = self._data_queries[i][INDEX_INPUT_VALUE]\n",
    "            for function in range(len(queries)):\n",
    "                values = queries[function]\n",
    "                self._data_results[(i * TOTAL_FUNCTIONS) + function, INDEX_FUNCTION_NUMBER] = function + 1\n",
    "                self._data_results[(i * TOTAL_FUNCTIONS) + function, INDEX_OUTPUT_VALUE] = self._data_observations[i][function]\n",
    "                for k in range(len(values)):\n",
    "                    self._data_results[(i * TOTAL_FUNCTIONS) + function, INDEX_WEEK_NUM] = self._data_queries[i][INDEX_WEEK_NUM]\n",
    "                    self._data_results[(i * TOTAL_FUNCTIONS) + function, INDEX_WEEK_NUM+1] = self._data_queries[i][INDEX_WEEK_NUM+1]\n",
    "                    self._data_results[(i * TOTAL_FUNCTIONS) + function, k + INDEX_FUNCTION_VALUE] = float(values[k])\n",
    "\n",
    "    def _format_data_file(self, filename):\n",
    "        student_id = \"\"\n",
    "        data_type = \"\"\n",
    "        pos = filename.name.find(DATA_STUDENT_ID_SEPARATOR1)\n",
    "        if pos == -1:\n",
    "            pos = filename.name.find(DATA_STUDENT_ID_SEPARATOR2)\n",
    "            if pos != -1:\n",
    "                student_id = int(filename.name[0:pos])\n",
    "        else:\n",
    "            student_id = int(filename.name[0:pos])\n",
    "        pos = filename.name.find(DATA_TYPE_OBSERVATIONS)\n",
    "        if pos != -1:\n",
    "            data_type = DATA_TYPE_OBSERVATIONS\n",
    "        else:\n",
    "            data_type = DATA_TYPE_QUERIES\n",
    "        return [student_id, data_type, filename]\n",
    "        \n",
    "    def _format_file_list(self):\n",
    "        table = []\n",
    "        total_weeks = 0\n",
    "        total_files = 0\n",
    "        for key, files in self._file_list:\n",
    "            total_weeks +=1\n",
    "            for i in range(len(files)):\n",
    "                total_files +=1\n",
    "                values = []\n",
    "                #append week number\n",
    "                values.append(key)\n",
    "                values.append(files[i][2])\n",
    "                table.append(values)\n",
    "        headers = [f'Week ({total_weeks})', f'Filename ({total_files})']\n",
    "        print(tabulate(table, headers=headers))\n",
    "        print(\"\\n\")\n",
    "\n",
    "    def _format_function_sample_total(self):\n",
    "        print(\"Total samples per function:\")\n",
    "        print(\"===========================\")\n",
    "        for function in self._update_data:\n",
    "            print(f'{TAB_SPACE}{COLOR_BLUE}Func {function} samples = {len(loader._update_data[function][\"outputs\"])}{RESET}')\n",
    "\n",
    "    def _save_updated_results(self, path, data, prefix, message):\n",
    "        print(message)\n",
    "        data_dir = self._create_updated_directory(os.getcwd() + path)\n",
    "        for key, value in data.items():\n",
    "            function_dir_path = f'{path}{FUNCTION_DIRECTORY}{key}'\n",
    "            function_dir = self._create_updated_directory(f'{data_dir}/{FUNCTION_DIRECTORY}{key}')\n",
    "            input_file = f'{function_dir}/{prefix}{FILENAME_INPUTS}'\n",
    "            output_file = f'{function_dir}/{prefix}{FILENAME_OUTPUTS}'\n",
    "            np.save(input_file, value[\"inputs\"])\n",
    "            np.save(output_file, value[\"outputs\"])\n",
    "            print(f'{BLANK_SPACE}{COLOR_BLUE}.{function_dir_path}/{prefix}{FILENAME_INPUTS}{RESET}')\n",
    "            print(f'{BLANK_SPACE}{COLOR_BLUE}.{function_dir_path}/{prefix}{FILENAME_OUTPUTS}{RESET}')\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98bca7d8-04c7-4e55-9d18-a21ea1b58f95",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create instance of Loader\n",
    "loader = CapstoneResultLoader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba470526-7142-45a0-97e8-8755add27e08",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.get_initial_function_data1()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be22fe4a-7868-464a-b2f8-e152b22b4a8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.get_initial_function_data2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6212ffcd-4434-422c-8fac-41c10ec392b7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# loader.get_initial_function_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73cfba29-9888-4037-9dd3-f8baa1542110",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.print_student_results(484, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f850fec-15fc-45f9-aa46-249a08a7b218",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.print_function_results(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca998723-6961-4691-8aae-89485a3a8cd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.plot_observations(484)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd79039c-8f80-46c9-8a17-bad938ac62c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.print_initial_function_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c07bb5f-ba77-4f48-a611-26383e0c126a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.print_function_sample_total()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34b6ed31-2180-4555-a23b-6a592f5bb8c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.get_update_function_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23ad1406-abdd-4eb7-9f4a-28bd4faef198",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.save_plot_observations()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d83d404-45b3-4296-82db-f536208b3ce1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.print_function_sample_total()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16b4fe5c-98c8-4ab7-92a0-204324fa5f1a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}