EMA_mine.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import lightning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EMA_linear(nn.Module):\n",
    "    \"\"\"EMA_linear\n",
    "\n",
    "    Exponential Moving Average Linear Layer\n",
    "\n",
    "    This layer is used to store a non-trainable version of the weights and biases of\n",
    "    another layer, so that it can act as a target layer for a twin layer that is actually\n",
    "    being optimized. The weights and biases of this layer are updated with an exponential\n",
    "    moving average upon calling the `update` method. The update is performed every `steps_to_update`\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        in_features,\n",
    "        out_features,\n",
    "        alpha=0.999,\n",
    "        negative_slope=1e-3,\n",
    "        steps_to_update=1,\n",
    "    ):\n",
    "        \"\"\"__init__ EMA_linear\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        in_features : int\n",
    "            Number of input features\n",
    "        out_features : int\n",
    "            int\n",
    "        alpha : float, optional\n",
    "            EMA smoothing coefficient, by default 0.999\n",
    "        negative_slope : int, optional\n",
    "            leaky_ReLU negative slope used to initialize the weights, by default 1e-3\n",
    "        steps_to_update : int, optional\n",
    "            steps before an actual update is performed, by default 1\n",
    "        \"\"\"\n",
    "        super(EMA_linear, self).__init__()\n",
    "\n",
    "        assert 0.0 <= alpha <= 1.0, f\"Invalid alpha: {alpha}\"\n",
    "        assert steps_to_update > 0, f\"Invalid steps_to_update: {steps_to_update}\"\n",
    "\n",
    "        self.alpha = alpha\n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.steps_to_update = steps_to_update\n",
    "        self._step = 0\n",
    "        self.negative_slope = negative_slope\n",
    "        self.register_buffer(\"weight\", torch.zeros(out_features, in_features))\n",
    "        self.register_buffer(\"bias\", torch.zeros(out_features))\n",
    "        self.reset_parameters()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        nn.init.kaiming_normal_(self.weight, a=self.negative_slope)\n",
    "        nn.init.constant_(self.bias, 0)\n",
    "\n",
    "    def set_parameters(self, weight, bias):\n",
    "        \"\"\"set_parameters\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        weight : torch.Tensor\n",
    "            weight tensor\n",
    "        bias : torch.Tensor\n",
    "            bias tensor\n",
    "        \"\"\"\n",
    "        with torch.no_grad():\n",
    "            self.weight.copy_(weight)\n",
    "            self.bias.copy_(bias)\n",
    "\n",
    "    def forward(self, input):\n",
    "        \"\"\"forward\n",
    "\n",
    "        MX + b, that's all\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        input : torch.Tensor\n",
    "            input tensor\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        torch.Tensor\n",
    "            output tensor\n",
    "        \"\"\"\n",
    "        return F.linear(input, self.weight, self.bias)\n",
    "\n",
    "    def update(self, weight, bias):\n",
    "        \"\"\"update\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        weight : torch.Parameter\n",
    "            weight tensor\n",
    "        bias : torch.Parameter\n",
    "            bias tensor\n",
    "        \"\"\"\n",
    "\n",
    "        self._step += 1\n",
    "\n",
    "        if self._step % self.steps_to_update != 0:\n",
    "            return\n",
    "\n",
    "        with torch.no_grad():\n",
    "            self.weight.copy_(self.alpha * self.weight + (1 - self.alpha) * weight)\n",
    "            self.bias.copy_(self.alpha * self.bias + (1 - self.alpha) * bias)\n",
    "        self._step = 0\n",
    "\n",
    "    def extra_repr(self):\n",
    "        return \"in_features={}, out_features={}, alpha={}\".format(\n",
    "            self.in_features, self.out_features, self.alpha\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AnomalyDataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, N, D):\n",
    "        super(AnomalyDataset, self).__init__()\n",
    "        self.normal_data = torch.randn(N - N // 10, D) - 2.5 * torch.randn(D)\n",
    "        self.anomaly_data = torch.randn(N // 10, D) + 5.0 * torch.randn(D)\n",
    "\n",
    "        # introduce correlation between the two dimensions\n",
    "        for i in range(1, D):\n",
    "            for j in range(i):\n",
    "                self.anomaly_data[:, i] += self.anomaly_data[:, j] * torch.randn(\n",
    "                    1\n",
    "                ) + torch.sin(self.anomaly_data[:, j])\n",
    "                self.normal_data[:, i] += self.normal_data[:, j] * torch.randn(\n",
    "                    1\n",
    "                ) + torch.cos(self.normal_data[:, j])\n",
    "\n",
    "        self.total_data = torch.cat([self.normal_data, self.anomaly_data], dim=0)\n",
    "        self.labels = torch.cat([torch.zeros(N - N // 10), torch.ones(N // 10)], dim=0)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.total_data)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.total_data[idx], self.labels[idx]\n",
    "\n",
    "\n",
    "class AnomalyModule(lightning.LightningDataModule):\n",
    "\n",
    "    def __init__(self, N, D, batch_size=32):\n",
    "        super(AnomalyModule, self).__init__()\n",
    "        self.data = AnomalyDataset(N, D)\n",
    "        self.train_data, self.val_data = torch.utils.data.random_split(\n",
    "            self.data, [int(0.8 * N), N - int(0.8 * N)]\n",
    "        )\n",
    "        self.batch_size = batch_size\n",
    "\n",
    "    def setup(self, stage=None):\n",
    "        pass\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(\n",
    "            self.train_data, batch_size=self.batch_size, shuffle=True\n",
    "        )\n",
    "\n",
    "    def val_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(\n",
    "            self.val_data, batch_size=self.batch_size, shuffle=False\n",
    "        )\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "\n",
    "def contrastive_loss(y_hat_1, y_hat_2, temperature=0.1):\n",
    "\n",
    "    y_hat_1 = F.normalize(y_hat_1, dim=0)\n",
    "    y_hat_2 = F.normalize(y_hat_2, dim=0)\n",
    "\n",
    "    return 2 - 2 * (y_hat_1 * y_hat_2).sum(dim=0).mean()\n",
    "\n",
    "\n",
    "class SiameseProjectionHead(lightning.LightningModule):\n",
    "\n",
    "    def __init__(self, in_features, out_features):\n",
    "        super(SiameseProjectionHead, self).__init__()\n",
    "        self.head_1_1 = nn.Linear(in_features, out_features)\n",
    "        self.head_1_2 = nn.Linear(out_features, out_features)\n",
    "        self.head_2_1 = EMA_linear(in_features, out_features, steps_to_update=2)\n",
    "        self.head_2_2 = EMA_linear(out_features, out_features, steps_to_update=2)\n",
    "\n",
    "        self.similarities = []\n",
    "\n",
    "        self.apply(self._init_weights)\n",
    "\n",
    "        # self.head_2_1.set_parameters(self.head_1_1.weight, self.head_1_1.bias)\n",
    "        # self.head_2_2.set_parameters(self.head_1_2.weight, self.head_1_2.bias)\n",
    "\n",
    "    def _init_weights(self, m):\n",
    "\n",
    "        if isinstance(m, nn.Linear):\n",
    "            nn.init.kaiming_normal_(m.weight, a=1e-3)\n",
    "            nn.init.constant_(m.bias, 0)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        y_hat_1 = self.head_1_2(F.leaky_relu(self.head_1_1(x), negative_slope=1e-3))\n",
    "        y_hat_2 = self.head_2_2(F.leaky_relu(self.head_2_1(x), negative_slope=1e-3))\n",
    "\n",
    "        return y_hat_1, y_hat_2\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "\n",
    "        x, _ = batch\n",
    "        y_hat_1, y_hat_2 = self(x)\n",
    "\n",
    "        loss = contrastive_loss(y_hat_1, y_hat_2)\n",
    "\n",
    "        self.head_2_1.update(self.head_1_1.weight, self.head_1_1.bias)\n",
    "        self.head_2_2.update(self.head_1_2.weight, self.head_1_2.bias)\n",
    "\n",
    "        if batch_idx % 100 == 0:\n",
    "            # log cosine similarity of weights\n",
    "            with torch.no_grad():\n",
    "                self.similarities.append(\n",
    "                    F.cosine_similarity(self.head_1_1.weight, self.head_2_1.weight)\n",
    "                    .mean()\n",
    "                    .item()\n",
    "                )\n",
    "\n",
    "        return loss\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        return torch.optim.AdamW(self.parameters(), lr=1e-5)\n",
    "\n",
    "    def anomaly_score(self, x):\n",
    "\n",
    "        y_hat_1, y_hat_2 = self(x)\n",
    "        return contrastive_loss(y_hat_1, y_hat_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = AnomalyModule(5000, 128, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    }
   ],
   "source": [
    "net = SiameseProjectionHead(128, 64)\n",
    "\n",
    "trainer = lightning.Trainer(max_epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name     | Type       | Params\n",
      "----------------------------------------\n",
      "0 | head_1_1 | Linear     | 8.3 K \n",
      "1 | head_1_2 | Linear     | 4.2 K \n",
      "2 | head_2_1 | EMA_linear | 0     \n",
      "3 | head_2_2 | EMA_linear | 0     \n",
      "----------------------------------------\n",
      "12.4 K    Trainable params\n",
      "0         Non-trainable params\n",
      "12.4 K    Total params\n",
      "0.050     Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2773c39fa2ad4b02b5b05c9e9c0d15b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=100` reached.\n"
     ]
    }
   ],
   "source": [
    "trainer.fit(net, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGxCAYAAACeKZf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABS20lEQVR4nO3deXxM9/4G8GdmskhiQkhkI8SSxBoSKbEFoY1qURSlaEtVadVSNPbErkTtVC2xtm51s9QSWoqEin0nEiX7npA9+f7+8Ovczk1ohpk5mcnzfr2+L3LmnJMnH249d+bMGRkAASIiIiIjIZc6ABEREZE2sdwQERGRUWG5ISIiIqPCckNERERGheWGiIiIjArLDRERERkVlhsiIiIyKiw3REREZFRMpA4gBScnJ2RnZ0sdg4iIiDSgVCoRFxf3r/tVunLj5OSE2NhYqWMQERHRC3B2dv7XgiNpuenYsSMmT54Mb29vODk5oU+fPvj555+fe4yfnx9CQkLQtGlTPHz4EPPmzUNoaGi5v+ffz9g4Oztr/dkbpVKJ2NhYnZyb1HHW+sNZ6w9nrT+ctf5oa9Z/n6c855C03FhZWeHy5cvYvHkzfvzxx3/dv169ejhw4ADWr1+PIUOGwN/fH9988w3i4+Nx5MgRjb53dna2zv5C6/LcpI6z1h/OWn84a/3hrPVHn7OWtNwcOnQIhw4dKvf+o0ePRnR0ND7//HMAwK1bt9ChQwdMmDBB43JDRERExsmgrrnx9fVFWFiY2rbDhw/jq6++euYxZmZmMDc3V32tVCrVftUmXZ6b1HHW+sNZ6w9nrT+ctf5oa9aaHG9Q5cbBwQGJiYlq2xITE1GtWjVUqVIFeXl5pY4JDAzEnDlzSm3X5UXFvGBZfzhr/eGs9Yez1h/OWn/0OWuDKjcvYuHChQgJCVF9rcuLyHiBmv5w1vrDWesPZ60/nLX+aPuC4vIwqHKTkJAAe3t7tW329vbIzMws81kbACgoKEBBQUGp7byg2Dhw1vrDWesPZ60/nLX+6HPWBnWH4vDwcPj7+6tt6969O8LDwyVKRERERBWNpOXGysoKnp6e8PT0BAC4urrC09MTderUAQAsWLBA7R4269evR/369bF48WK4u7vj448/xoABA7B8+XJJ8hMREVHFJKRafn5+oixbtmwRAMSWLVvEb7/9VuqYCxcuiLy8PHHv3j0xfPhwjb6nUqkUQgihVCq1/vPo8txcnDVnbfyLs+asjXFpa9aanEfSa25OnDgBmUz2zMfff//9Mo/x8vLSZSwiIiIyYAZ1zQ0RERHRv2G5ISIiIqPCckNERERGheWGiIiItEKuUMC6lh1qODtKmsOgbuJHRERE0rCqXg3WtexQzd4O1Wr9d1nXsoW1rS2sa9miag0byOVy3In4Exs+HCdZVpYbIiKiSs6ymjVsHB1Q3aEWqjvYq36tZl/raYmxt4PpPz6E+nmKi4ogl0v7whDLDRERkRGTyeWwtqsJG0dH1HB2gI2jI2ycHJ6WGUd72Djaw9zSslznyk5NQ2ZiMrKSU5CZlIyspGRkJqcgKynl6bbkZDxJz4QoKdHxT/V8LDdEREQGzrKaNWrWdkbN2k6oUdsZNWo7oqazE2ycnhYZE1PTfz1HdmoaMhISkZGQhIyERGQmJiEjMVn1+8ykFBQXFurhp3l5LDdEREQVnEwmg3UtW9jWqQ1bl9qoWac2atZxhm2d2qhZ2wkW1srnHl9cWISMxESkxyUgPT4BabHxyIhPRHr8068zEpNRlJ+vp59G91huiIiIKgjLataoVa8u7FxdYFfXBXZ168C2bh3Y1qkNM4sqzz02MzEZqY9ikRYbr/o1LS4eaY/ikJWcgpLiYj39FNJjuSEiItIjmVyOGs5OsK9fD7Vc66otq+rVnnlccVER0mLjkfLwEVIfxiLlr6e/pj58hNTYeKN65uVlsdwQERHpgFyhQM06znBo4Ar7hvXhUL8e7Bu4wq6ey3PfeZQen4DkmL+Q/ODh01//eoiUBw+RFhePkqLK8+zLy2C5ISIieknWtezg2KgBnNwawNGtIRwa1kct17rPLDGFeflIinmApPsxSIp+8HTF/IXkB3+hMI/PwLwslhsiIqJyUpiawqGBK5zcG8LJ3Q1O7g3h6NbwmS8nFeTmIfF+NBLuRSMx6j4SomKQeD8a6XEJkr9d2pix3BAREZXBzMICzh6N4NzYHc6N3eDs4QaHBvWhMC39T2dxURGSHzxE/J17iL8bhYR795Fw9z7SYuMghJAgfeXGckNERJWeaRVzOHu4o05TD9Ru6oHaTTxQy7VumXfazcnMQtztu4i9fRfxt+8i7vY9JN6PQVFBgQTJqSwsN0REVKnI5HLYN3DFlbREvDl1PBzcn14jozAp/U9iZmIyHt28jdhbdxB78zZib95BenyCBKlJEyw3RERk1KxsqqOeZzO4tGiGui2aok6zxqhiZYWwuGh4935dtV9WcgoeXruJhzdu4dH1W3h04xayU9MkTE4viuWGiIiMhkwmQ6369eDaqgXqtWyOep7NYVfPpdR++U+eoGEtR+xa9zXuRV7Cw+s3kJmYLEFi0gWWGyIiMlhyEwVqN/FAfa+WcPVqAddWnmW+cykhKhoPLl3FgyvX8ODKdeQkpyIzIwMj1m9Gdna2BMlJl1huiIjIYChMTFCnWRM08GmFBq1boV7L5qU+0To/Jxd/Xb2OmEtXEXPpCh5cuY7cLPUCo1Q+/7OYyLCx3BARUYUlVyjg7OGGhm280egVb9Rr5QlzSwu1fZ5kZCL6wiXcj7yM+xcuI/bWbd7Jt5JjuSEiogrFtm4duPu+AjdfHzRo7VXqE6+zU9NwP/ISos5fRNT5i0i8d5/3kiE1LDdERCQpC2slGrVpDY/2beHW7hXYODqoPZ6blY2o8xdw92wk7p2LRMK9+xIlJUPBckNERHolk8ng3NgdHh194dG+Leq2aAq5QqF6vKigANEXr+BuxHncCT+HRzdv86MKSCMsN0REpHPmVpZwb9cGjTu2g0dHX1jb1lR7PCEqGrfPnMXt02cRfeESCnLzJEpKxoDlhoiIdMLGyQFNO3dA084d0aC1l9pnMuU9eYK7Eedx61Q4bp8+y7v+klax3BARkdY4e7ihmb8fmnXpCCf3RmqPJUU/wM0/zuDmyTO4f+EyigsLJUpJxo7lhoiIXphMLodrqxb/X2g6oWZtJ9VjJcXFiL54Bdd/+wPXT5xCyoOHEialyoTlhoiINCKTy1HfyxOer/mjub8frO1sVY8V5Obh1ukIXDt2Ajf/OIOczCwJk1JlxXJDRET/SiaTwdXLEy0DuqFF9y5Q1qyheiwnKws3fj+Nq8d+x+0zZ1GYly9hUiKWGyIieo46TRuj1evd4fmaP6rb11Jtz8nMwtVjJ3Dl6HHcjTiP4qIiCVMSqWO5ISIiNbYuteHV8zV49XwNdnXrqLbnZmXj6rETuHQoDHfPnedHHFCFxXJDRESwql4NrV7vDq+eAajboqlqe35OLq7//gcu/XoUt06f5TucyCCw3BARVVJyEwUad/BF69490cSvPUxMTQEAxUVFuBPxJy7sP4Rrx/9AQW6uxEmJNMNyQ0RUydRyrYs2fXvB+80AtQuDH964hchffsXFQ0fxODVdwoREL4flhoioEjCzsEDL1/zRpl8v1GvZXLU9KyUVF/Yfxp+/HETC3SgJExJpD8sNEZERc3RriHYD3oJXz9dQpaoVgKcvO908eRpnf9iPW6fCUVLMC4PJuLDcEBEZGRNzc7QK8Efbt/ugnud/n6VJfvAQZ3/4Bed/+RXZKakSJiTSLZYbIiIjUcPZEe0G9MUrfd+EVfVqAIDiwiJcPfY7zuz5EVF/XpA4IZF+sNwQERk4N99X0OGd/mjs1x5yuRwAkPooDhHf/4RzP+3nxcFU6bDcEBEZINMq5vB+swc6DhkAhwauqu23TkXg9Ld7cfOPMxAlJRImJJIOyw0RkQGxrmWH9oP6wfftPqqXnvIeP8G5H/fj9Hd7+cnbRGC5ISIyCA4N66Pze4PR6vVXVTfbS3n4CKd2/gfnftqP/Cc5EickqjhYboiIKrAGPl7o8v4QNO7YTrUtKvIiTm77Ftd/P8WXnojKwHJDRFTByGQyNOvaCV1HDFN9zlNJcTGuHjuB37bsxMNrNyROSFSxsdwQEVUQcoUcNzKS8fH2DahVvx4AoDAvH+d+2o8TobuR+ihW2oBEBoLlhohIYgoTE7Tu1QPdR72PQ4+iUKt+PeRmP8bpb/fijx3f4XEa38pNpAmWGyIiiShMTeHTpyf8RwxDDWdHAICFwgT7Vn+N37btQt7jJxInJDJMLDdERHqmMDGBz1tvoNuHw2Hj6AAAyEpOwZld3+PAhk2Yuv1bFhuil8ByQ0SkJ3KFAt5vBqD7Rx+gZm0nAEBmYjKOb96OiL2/wMLMDKYbt0obksgIsNwQEemYTCZDy4BueG3MSNjVcwEAZKWk4tjGUER8/zOKCgqe7mhmJmFKIuPBckNEpEMeHdri9c8+hrOHGwDgSXoGjm/ajtPf7UVhXr7E6YiME8sNEZEOuLRoip7jx6ChjxcAIDf7MX4P3YU/tn+H/BzeTZhIl1huiIi0qGad2ug5/mN4vtoVAFCYn4/Tu/fi2DehyMnMkjgdUeXAckNEpAVW1auh20fvo/3AflCYmqCkuBh//nQAR9ZtQkZiktTxiCoVlhsiopegMDVFxyED0O3D4bCwVgIAbpw8jf0ha5AYFS1xOqLKieWGiOgFNe3SEb0+Hwdbl9oAgNhbd7Bv2WrcjfhT4mRElZtc6gBjxoxBdHQ0cnNzERERAR8fn+fu/9lnn+HWrVvIycnBX3/9hZCQEJibm+spLRER4NCwPj7auBIfrFwCW5fayEpOwbcz52H5wPdZbIgqAEmfuRkwYABCQkIwevRonD17FuPHj8fhw4fh7u6O5OTkUvu/8847WLRoET744AOcOXMGbm5u2Lp1K4QQmDRpkgQ/ARFVJlWUVdHjk1FoN7Av5AoFCvPzcWLbtzj+zTa+A4qoApG03EycOBEbN27E1q1bAQCjR49Gz5498cEHH2Dx4sWl9m/Xrh1Onz6N3bt3AwAePHiA3bt3o02bNvqMTUSVjEwmQ+ver6Pn+DFQ1qwBALh85Dj2h6xGWmy8xOmI6H9JVm5MTU3h7e2NhQsXqrYJIRAWFgZfX98yjzlz5gzeffdd+Pj44M8//4Srqytef/11bN++XV+xiaiScfZwQ9/pn6Ney+YAgISoaPy0MAR3z56XOBkRPYtk5cbW1hYmJiZITExU256YmAgPD48yj9m9ezdsbW1x6tQpyGQymJqaYt26dWoF6X+ZmZmpXZOjVCrVftUmXZ6b1HHW+lNZZ21uaYkuo4bjlX69IFcoUJCTi98378DZPT+iuKiI/w0xcJy1/mhr1pocb1DvlvLz88O0adMwZswYnD17Fg0bNsSKFSswY8YMzJs3r8xjAgMDMWfOnFLbY2NjdZZTl+cmdZy1/lSWWQshcDcrDb/Fx+BJUSEAwM26Bvzc6+KL9V2A9Zt0nqGyzLoi4Kz1R5+zlgEQevtu/2BqaoqcnBz0798fP//8s2r71q1bUb16dfTp06fUMSdPnkRERASmTJmi2jZkyBB8/fXXqFq1KoQo/aOU9cxNbGwsnJ2dkZ2drdWfSZfnJnWctf5UpllXc6iFnpM+gVv7tgCAtEexOLB0NaLORerl+1emWUuNs9Yfbc367/NYW1v/63kke+amsLAQkZGR8Pf3V5UbmUwGf39/rF69usxjLC0tUVJSoratuLhYdWxZ5aagoAAFf3/i7j9kZ2fr7C+0Ls9N6jhr/THmWcvkcnR4pz96jPsI5paWKCosxG+bdyBsYyiK8vX/4ZbGPOuKhrPWH33OWtKXpUJCQhAaGorz58/j3LlzGD9+PKysrLBlyxYAQGhoKGJjYzFt2jQAwL59+zBx4kRcvHhR9bLU3LlzsW/fvlKlh4ioPBwa1seAOYGo69kMABAVeRHfBy1GUvQDiZMR0YuStNzs2bMHdnZ2CA4OhoODAy5duoSAgAAkJT39HBYXFxe10jJv3jwIITBv3jw4OzsjOTkZ+/btw/Tp06X6EYjIQClMTdHtw+HoOnIYTExNkZv9GPuXr8HZ738u81lgIjIckl1zIxWlUomsrKxyvWZXkc5N6jhr/THGWddu4o5B82bCsVEDAMC14yewd/4yZCWVvnmoPhnjrCsqzlp/tDVrTc5jUO+WIiJ6GQpTU3T/6H10HTEUChMTZKem4YcFy3DlyHGpoxGRFrHcEFGl4NzYDe/Mn6V6tubSoTD8sGAZnqRnSBuMiLSO5YaIjJpcoUDXkcPw6kcfQGH6/8/WzF+KK0d/kzoaEekIyw0RGS3bunUweP4s1TuhLh85jr3zvuSzNURGjuWGiIyS74C38OakT2FuaYHcrGz8sGApLhw4InUsItIDlhsiMipVa9hg4NzpaNKpPQDgbsR5fDtjLjISkyRORkT6wnJDREbDvX1bDJo3A9a2NVGYn48DX63DqZ17eN8aokqG5YaIDJ7C1BQ9J4yB39BBAID4u1HYMXU2Eu5GSZyMiKTAckNEBs22bh0M+3IenBu7AQBO7foP9oWskeQzoYioYmC5ISKD5fXGa+g/cwrMLS3xOC0d386cj5snT0sdi4gkxnJDRAbHzKIK3gqchFfeegMAcPfseewKDEJWcorEyYioImC5ISKDYt/AFcOWzoNDw/ooKS7GkfWbEfb1Voh/fMguEVVuLDdEZDCevgw1FeaWFshMSsbOqbMRdf6i1LGIqIJhuSGiCs/EzAy9p3yGdgP7AgBunzmLXYFBeJyWLnEyIqqIWG6IqEKzcXLA8JAFqNO0MUpKShC2YQuOrN/Ml6GI6JlYboiownJv3xbvLg6CZTVrPMnIxM4v5uD26QipYxFRBcdyQ0QVkv/I4Qj4dBTkcjkeXLmObZOmIyMhUepYRGQAWG6IqEIxt7TEoPkz0aJbZwBA+Pc/4ccFISguLJQ2GBEZDJYbIqowbOvWwfsrFsOhgSuKCgrww4JlOLv3F6ljEZGBYbkhogrBvX1bDF0SDAtrJTISkxA6cRr+unJd6lhEZIBYbohIcn7D3sEbE8dCrlAg+sJlhE6chuzUNKljEZGBYrkhIsmYmJmh/6wp8OndEwBwdu8v2Dt/Ka+vIaKXwnJDRJJQ1qyB91YsQj3P5iguKsIvX67AqV3fSx2LiIwAyw0R6Z2jWwOMWL0UNo4OyMnMwrbPZ+BuxJ9SxyIiI8FyQ0R61bhjO7z7ZTCqWFkhKfoBNn3yOVL+eiR1LCIyIiw3RKQ3Hd8diF6ffwq5QoG7Z88jdOJ05GZlSR2LiIwMyw0R6ZxcocBbgRNVH3wZ8f3P2Dv/S5QUFUucjIiMEcsNEemUmYUFhi6diyad2qOkpAT7l63GiW27pY5FREaM5YaIdEZpWxMj1ixFnSYeKMzLx46ps3Ht+AmpYxGRkWO5ISKdsK9fDyPXhaCGkyMep6Vj06eTecdhItILlhsi0rr63i3xwcolsLBWIjnmL2z8eCJSH8VKHYuIKgmWGyLSqubdOmPIojkwNTdH9MUr2DJuCp5kZEodi4gqEZYbItKadgP74q1pkyCXy3H12AnsmDobRfn5UsciokqG5YaItCLg01HoPup9AMCZPT/ih/lLIUpKJE5FRJURyw0RvRS5QoF+Myajbf/eAIBDazbi6PrNEqciosqM5YaIXpjC1BTvLg5Ci+5dUFJcjL3zvkTE9z9LHYuIKjmWGyJ6IeaWlnhvxSK4tfVBUUEBtk+exXvYEFGFwHJDRBqzrGaNkWtDULdFU+Q9eYKtn32Bu2fPSx2LiAgAyw0Raci6lh0+2vAVHBrWx5P0DGz8eCIeXr8pdSwiIhWWGyIqtxrOjhj9zSrUrO2MjMQkfD3qMyTej5E6FhGRGpYbIiqXmi61MXTFIlS3r4WUvx5h/YefIj0uQepYRESlsNwQ0b9KznuC99cuQ9UaNkiIisaGD8chKzlF6lhERGViuSGi53Ju7I490TdQtYYNYm/ewYaPPsOT9AypYxERPRPLDRE9U72WLTBs5SLkFxfj4bUbWD/qM+RlP5Y6FhHRc8mlDkBEFVP91q0wasNymFtZobaVNbZ/9gWLDREZBJYbIiql4Sve+HBtCMwtLRF17gLequuOgtw8qWMREZULyw0RqXHzfQUj1yyDmUUV3DwVjt1TZsFUrpA6FhFRubHcEJGKR4e2+GDVEphWMcf1309hy7ipKCookDoWEZFGNC43c+bMgYuLiy6yEJGE3Nu3xfsrFsPU3BxXj51A6IRAFBcWSh2LiEhjGpeb3r17IyoqCmFhYXjnnXdgZmami1xEpEfu7drg/RWLYGJmhithv2Pb59NRXFQkdSwioheicblp1aoVfHx8cP36daxYsQIJCQlYu3YtWrdurYt8RKRjbr6v4P2V/33GZsfkmSgpKpY6FhHRC3uha24uXbqEzz77DE5OThgxYgRq166N06dP4/Llyxg3bhysra21nZOIdKBRWx98sHIJTM3Nce23k9j++Qw+Y0NEBu+lLiiWyWQwNTWFmZkZZDIZ0tPT8cknn+Dhw4cYMGCAtjISkQ40aN3qabGpYo7rv/2BbRP5UhQRGYcXKjdeXl5YtWoV4uPjsXz5cly8eBGNGzdG586d4ebmhunTp2PlypXazkpEWlLPszlGrFkKM4squHHiNEInsdgQkfHQuNxcuXIFERERcHV1xYgRI1CnTh0EBgYiKipKtc/u3bthZ2en1aBEpB11mjbGyHVPb9B3+8xZhE6cxndFEZFR0fizpfbs2YPNmzcjLi7umfukpqZCoeBNv4gqGke3hhi14StYKKvi3p8XsOUz3seGiIyPxs/c/H1tzf+qUqUKZs6cqZVQRKR99vXrYfTGlbCsZo2YS1ex+ZPJKMzLlzoWEZHWaVxuZs+ejapVq5babmlpidmzZ2slFBFpV83azvho40pUrWGDv67dwMaPJyA/J0fqWEREOvFCz9wIIUpt9/T0RFpamlZCEZH2VLO3w0cbV6JaLTvE343CxtETkPf4idSxiIh0ptzlJi0tDampqRBC4M6dO0hNTVWtjIwMHD16FHv27NE4wJgxYxAdHY3c3FxERETAx8fnuftXq1YNq1evRlxcHPLy8nD79m306NFD4+9LVBlY2VTHR1+vRM3aTkh+8BAbPhyHnMwsqWMREelUuS8oHj9+PGQyGTZv3ozZs2cjMzNT9VhBQQFiYmIQERGh0TcfMGAAQkJCMHr0aJw9exbjx4/H4cOH4e7ujuTk5FL7m5qa4ujRo0hKSkL//v0RGxuLunXrIiMjQ6PvS1QZVFFWxaj1X8G+fj2kxydgw4fjkJ3KZ1eJqHIQmqxOnToJExMTjY551oqIiBCrVq1SfS2TycSjR4/E1KlTy9z/o48+Evfu3Xup769UKoUQQiiVSq38DPo6Nxdnrckys6giPgldL5ZdDRdzfj8g7Oq5cNYGsDhrztoYl7Zmrcl5yvXMjVKpRHZ2NgDg4sWLsLCweOa+f+/3b0xNTeHt7Y2FCxeqtgkhEBYWBl9f3zKP6dWrF8LDw7FmzRr07t0bycnJ2LVrFxYvXoySkpIyjzEzM4O5ubnaz/LPX7VJl+cmdZz1sylMTDBocRBcvTyRm5WNHROmIS81/YVnxVnrD2etP5y1/mhr1pocX65yk56eDkdHRyQnJyMjI6PMC4r/vtDYxKR8r3TZ2trCxMQEiYmJatsTExPh4eFR5jH169dH165dsXPnTrz++uto2LAh1q5dC1NTUwQHB5d5TGBgIObMmVNqe2xsbLlyvghdnpvUcdbqhBA4+OgebmemwkQmx/st22L6hUtaOTdnrT+ctf5w1vqjz1mXq4l07dpV9U6oLl266DTQ88jlciQlJWHUqFEoKSnBhQsX4OzsjMmTJz+z3CxcuBAhISGqr5VKJWJjY+Hs7FzuZ5nKS5fnJnWcddl6TBiDNm/3QXFREXZMmY0ZEX++9Dk5a/3hrPWHs9Yfbc367/OUR7nKzcmTJwEACoUCfn5+2Lx580s3sJSUFBQVFcHe3l5tu729PRISEso8Jj4+HoWFhWovQd28eROOjo4wNTVFYRm3kC8oKEBBGXdgzc7O1tlfaF2em9Rx1v/VffQHaPN2H5SUlGD3tGBcPHpcq+fnrPWHs9Yfzlp/9Dlrje5zU1xcjMmTJ5f7pafnKSwsRGRkJPz9/VXbZDIZ/P39ER4eXuYxp0+fRsOGDSGTyVTb3NzcEBcXV2axIaos2g3si4CxHwIAflq0HBd/PSpxIiIi6Wh8E7/jx4/Dz89PK988JCQEH374IYYNGwYPDw+sW7cOVlZW2LJlCwAgNDQUCxYsUO2/bt061KhRAytWrECjRo3w+uuvY9q0aVizZo1W8hAZohavdsVb0yYBAA6v2YjTu7+XOBERkbQ0fgrm119/xaJFi9C8eXNERkbiyRP1O53u27ev3Ofas2cP7OzsEBwcDAcHB1y6dAkBAQFISkoCALi4uKi9BPXo0SO89tprWL58Oa5cuYLY2FisWLECixcv1vTHIDIKDXy8MGThbMjlcpz+di+OrN8sdSQiogpBo/eZFxcXP3MVFRVJ/n76f1u8z41xLM4awtGtoZh35qhYdjVcDFs2X8jkcs7awBdnzVkb46qw97n5J4VCoekhRKRlNk4OGLV+OSyUVRF1/iJ2BQZBPONeT0RElY3G19wQkbSsqlfDqPVfwdrOFnF37mHzuCkoKuMdgUREldULve3J0tISfn5+cHFxgZmZmdpjq1at0kowIirNtIo5Plj9JWq51kVaXPzTT/jOfix1LCKiCkXjctOyZUscPHgQlpaWsLKyQlpaGmxtbZGTk4OkpCSWGyIdkcnlGLIoCPU8myMnMwsbR09AVnKK1LGIiCocjV+WWr58Ofbt2wcbGxvk5uaibdu2qFu3LiIjI/H555/rIiMRAegzdTya+/uhMD8fm8dNQVL0A6kjERFVSBqXm5YtW2LZsmUQQqC4uBjm5uZ49OgRpkyZonZPGiLSns7DB6PD4LcBALumBSP6wmWJExERVVwal5t/fvxBUlISXFxcAACZmZmoU6eOdtMREVoGdMObn38KAPj5yxW4ckS7H6tARGRsNL7m5uLFi/Dx8cG9e/dw4sQJBAcHw9bWFkOHDsW1a9d0kZGo0nL18sQ782cCAE5u/w4nt30rcSIioopP42dupk2bhvj4eADA9OnTkZ6ejnXr1sHOzg6jRo3SekCiysq2bh28v2IxTMzMcCXsd/yydKXUkYiIDILGz9xERkaqfp+cnIwePXpoNRARPb2Xzcg1y2BVvRoeXLmOXYFzeJM+IqJy4k38iCoYEzMzvL9iMezq1kHqozhsHjcZhXn5UsciIjIY5Xrm5sKFCxBClOuE3t7eLxWIqDKTyWQYNHc6XL08kZuVjU1jJ+FxarrUsYiIDEq5ys1PP/2k4xhEBACvjf0QrV5/FcWFRdg6IRCJ92OkjkREZHDKVW6Cg4N1nYOo0vN+IwDdP3ofAPCfoIW4dy7yX44gIqKy8JobogrAtVULDAgKBACEbQzFnz8flDgREZHhKtczN6mpqXBzc0NqairS0tKee/1NzZo1tRaOqDKoWdtZ9Zbvy0eO49CqDVJHIiIyaOUqNxMmTEB2djYAYPz48brMQ1SpVFFWxYg1S2FlUx1/XbuB3dODy33xPhERla1c5Wbbtm1l/p6IXpzcRIHhy+bDvn49ZCQkYvOnU/iWbyIiLdD4Jn5/s7OzQ61atSCXq1+2c/Xq1ZcORVQZ9Jk6AW6+ryA/JwebPpmM7JRUqSMRERkFjcuNl5cXQkND0bhxY8hkMrXHhBAwMXnhvkRUabQb2BftB/VDSUkJdn4xB3G370odiYjIaGjcRDZv3ow7d+5gxIgRSExM5PUBRBpq1KY1+nwxAQDw68r1uP7bHxInIiIyLhqXm/r166Nfv36IiorSRR4io2brUhvDls2HwsQE53/5Fcc3bZc6EhGR0dH4PjfHjh2Dp6enLrIQGbUqyqoYsXopLKtZI+byVfwnaJHUkYiIjJLGz9yMHDkSoaGhaNasGa5du4bCwkK1x/ft26e1cETGQiaXY+iSuajlWhfp8QnY8tlUFBUUSB2LiMgoaVxufH190b59e/To0aPUY7ygmKhsb0wYC48ObZGfk4vNn07hh2ESEemQxi9LrVq1Cjt27ICjoyMUCoXaYrEhKs37zR7o/N5gAMC3M+fxnVFERDqmcbmpWbMmli9fjqSkJF3kITIqLs2b4O3ZUwEARzdswZUjxyVORERk/DQuNz/88AO6dOmiiyxERsW6lh3e+2oRTM3Nce34CRxes1HqSERElYLGryPduXMHCxcuRIcOHXD16tVSFxSvWrVKa+GIDJWJmRne/2oRqtWyQ/zdKOwK5GdGERHpywu9W+rx48fw8/ODn5+f2mNCCJYbIgD9Z02BS/MmeJKRic3jpiA/J0fqSERElcYL3cSPiJ6tw+C34dO7J0qKi7F98kykPYqTOhIRUaWi8TU3RPRsDXy80GvyOADAvpDVuBvxp8SJiIgqn3I9c7Ns2TLMnDkTOTk5WLZs2XP3nTRpklaCERkaG0cHDFs6DwoTE0TuP4ST276VOhIRUaVUrnLTqlUrmJqaqn7/LLxgkior0yrmeO+rRahawwYPb9zCnjn8aAUiIqmUq9x07dq1zN8T0VNvz/4CtZu4Izs1DVs/+wJF+flSRyIiqrRe+pobpVKJ3r17w93dXRt5iAxOx3cHwvuNABQXFWH75zOQkZAodSQiokpN43Lz3XffYezYsQCAKlWq4Pz589izZw+uXr2Kvn37aj0gUUXWoHUrvDnpEwDAvqWrEHX+osSJiIhI43LTqVMn/PHHHwCAt956CzKZDNWrV8e4ceMwY8YMrQckqqiq29fC0H9cQPzHzj1SRyIiIrxAualWrRrS0tIAAAEBAdi7dy9yc3Nx4MABNGrUSOsBiSoiEzMzDF++EMqaNRB78w7+E8QLiImIKgqNy83Dhw/h6+sLS0tLBAQE4MiRIwAAGxsb5OXlaT0gUUXUd9ok1R2It074AoV5vICYiKii0PgOxV999RV27tyJx48f48GDB/j9998BPH256urVq9rOR1ThtOnXC2369UJJcTF2TJmFtNh4qSMREdE/aFxu1q1bh7Nnz8LFxQVHjx5V3dvm/v37vOaGjF6dpo3Rd9rTG1X+uupr3Ak/J3EiIiL6XxqXGwC4cOECLly4oLbt4MGDWglEVFFZ2VTH8OULYGJmhqvHTuD4pm1SRyIiojLws6WIykEml+PdJcGwcXRAUvQDfDtjrtSRiIjoGVhuiMoh4JNRcGvrg/ycXGydEIi8x0+kjkRERM/AckP0L5p17YRuHw4HAOyZvQCJUdESJyIioudhuSF6DluX2hg0byYA4MT2b3HpUJjEiYiI6N+8ULnp0KEDtm/fjjNnzsDJyQkA8O6776J9+/ZaDUckJdMq5hi+fCEslFVxP/IS9oesljoSERGVg8blpm/fvjh8+DByc3PRqlUrmJubA3h65+Jp06ZpPSCRVPrPnAont4bISknF9skzUVJULHUkIiIqB43LzYwZMzB69GiMGjUKhYWFqu2nT5+Gl5eXVsMRSaXt233QulePp5/0PXkmspJTpI5ERETlpHG5cXd3x8mTJ0ttz8zMRPXq1bWRiUhSdZo2xltfTAAAHFyxHvf5Sd9ERAZF43KTkJCAhg0bltreoUMH3L9/XyuhiKRiWc0aw0Lmq27U9/vWnVJHIiIiDWlcbjZu3IgVK1bglVdegRACTk5OGDx4MJYuXYp169bpIiORXshkMgxeOBs1nByR/OAhb9RHRGSgNP74hUWLFkEul+PYsWOwtLTEyZMnkZ+fj6VLl2L1ar6bhAxX15HD0LhjOxTm5SN04jTeqI+IyEC90GdLLViwAF9++SUaNmyIqlWr4saNG3jyhP8QkOFq1KY1AsZ+CADYO/9LxN+5J3EiIiJ6US9UbgCgsLAQN2/e1GYWIklY17LDkMVBkCsUOPvDPvz50wGpIxER0UvQuNxYWlriiy++gL+/P2rVqgW5XP2ynQYNGmgtHJGuyU0UGLokGMqaNRB3+y5+WLBM6khERPSSNC4333zzDfz8/LB9+3bEx8dDCKGLXER68fq4j1HfuyVysx8jdOI0FOXnSx2JiIi0QGiy0tPTRbt27TQ65t/WmDFjRHR0tMjNzRURERHCx8enXMcNHDhQCCHEjz/+WO7vpVQqhRBCKJVKrf4Muj43l/Zn3bRLR7HsarhYdjVcNO/WWfKfqaIu/r3mrI1xcdaGN2tNzqPxW8HT09ORlpam6WHPNGDAAISEhCAoKAheXl64fPkyDh8+DDs7u+ceV7duXSxdurTMGwoS/Zsazo4YNG8GgKcfiHk17HdpAxERkdZoXG5mzpyJ4OBgWFhYaCXAxIkTsXHjRmzduhU3b97E6NGjkZOTgw8++OCZx8jlcuzcuROzZ8/mjQNJYwpTUwxdOg+W1tZ4cPkaDoSskToSERFpkcbX3EyaNAkNGjRAYmIiYmJi1D5fCgC8vb3LfS5TU1N4e3tj4cKFqm1CCISFhcHX1/eZx82aNQtJSUnYvHkzOnbs+NzvYWZmpvpwTwBQKpVqv2qTLs9N6l5m1j0mjIFLsybIyczCD3MWwlJLRd1Y8e+1/nDW+sNZ64+2Zq3J8RqXm59++knTQ57J1tYWJiYmSExMVNuemJgIDw+PMo9p3749RowYgZYtW5brewQGBmLOnDmltsfGxmoat9x0eW5Sp+ms72SmYv/DuwCAwS18MONulC5iGSX+vdYfzlp/OGv90eesNS43wcHBushRLlWrVsX27dvx4YcfIjU1tVzHLFy4ECEhIaqvlUolYmNj4ezsjOzsbK3m0+W5Sd2LzLpGbSd8tGUNzK2s8Me2bzFn/WYdpzQO/HutP5y1/nDW+qOtWf99nvJ44Zv4aUNKSgqKiopgb2+vtt3e3h4JCQml9m/QoAFcXV2xb98+1ba/77NTWFgId3f3UtfgFBQUoKCgoNS5srOzdfYXWpfnJnXlnbWJmRn6BU+DuZUVoiIv4peQ1SgpLtZDQuPBv9f6w1nrD2etP/qcdbnKTWpqKtzc3JCamoq0tLTn3tumZs2a5f7mhYWFiIyMhL+/P37++WcATz+80N/fv8zPqbp16xaaNWumtm3evHlQKpX47LPP8PDhw3J/b6pcek8dD2cPN2SnpmHH5FksNkRERqxc5WbChAmqtjVhwgSt3rgvJCQEoaGhOH/+PM6dO4fx48fDysoKW7ZsAQCEhoYiNjYW06ZNQ35+Pq5fv652fEZGBgCU2k70t1Y9uqPdgLdQUlKCXYFByEpOkToSERHpULnKzbZt21S/Dw0N1WqAPXv2wM7ODsHBwXBwcMClS5cQEBCApKQkAICLiwtKSkq0+j2p8rCr54L+s6cCAI5tDMWd8HMSJyIiIn3Q6A6BrVq1Es2aNVN93atXL/Hjjz+K+fPnC1NTU8nvhPhvi3coNo5VnlmbmJuLSd9vE8uuhouPN68RcoVC8tyGuPj3mrM2xsVZG96sdXqH4g0bNsDNzQ0A4Orqiu+++w45OTl4++23sWTJEk1PR6QzfaaOh5N7o6fX2UzhdTZERJWFxuXGzc0Nly5dAgC8/fbbOHHiBIYMGYL33nsP/fr103Y+ohfSqkd3+L7d5/+vs5mD7JTy3TqAiIgMn8blRiaTqd5+3a1bNxw8eBAA8PDhQ9ja2mo3HdELsK1bR3WdTdjXW3En/E+JExERkT5pXG7Onz+PGTNm4N1334Wfnx8OHDgA4OlLVP97p2EifTMxM8OwpfNQxcoK9/68gCPrNkkdiYiI9EzjcjN+/Hh4eXlh9erVmD9/PqKint6+vn///jhz5ozWAxJpoveUz1T3s9k5dTYE32lHRFTpaHyH4qtXr6JFixaltk+ePBnFvGCTJOT5mj/aDezL+9kQEVVyL/zxC15eXmjcuDEA4MaNG7h48aLWQhFpqmad2hgwJxAAcPybbbyfDRFRJaZxubGzs8N3330HPz8/1d2Bq1evjt9++w2DBg1CSgr/3zLpl8LUFEOXzkWVqk8/N+rw2m+kjkRERBLS+JqbVatWoWrVqmjatClq1qyJmjVrolmzZrC2tsbKlSt1kZHoud6YOBZ1mnjgSXoGdk6dzfvZEBFVcho/cxMQEIBu3brh1q1bqm03b97E2LFjceTIEa2GI/o3Hp3aodO7AwEAu6YHIzMxWeJEREQkNY2fuZHL5SgsLCy1vbCwUHX/GyJ9yCrIR+9pkwAAv23ZiVt/hEuciIiIKgKN28jx48exYsUKODo6qrY5OTlh+fLlOHbsmFbDET2LXKHAgYd3YWGtxIPL1/DryvVSRyIiogpC43LzySefwNraGjExMbh37x7u3buH6OhoWFtb49NPP9VFRqJSun70HuJzHyMv+zG2T5mJ4qIiqSMREVEFofE1N48ePYKXlxe6desGDw8PAE+vueGzNqQv7u3bosP/X2fz84JlSI9LkDgRERFVJC98n5uwsDCEhYVpMwvRv7K2s8XgBbMAAJ417DHnxGmJExERUUVT7pelunTpguvXr0OpVJZ6zNraGteuXUOHDh20Go7on2RyOQYvnI2qNWwQf+ce/BzqSh2JiIgqoHKXm/Hjx2Pjxo3Izs4u9VhWVhY2bNiAiRMnajUc0T91G/UeGrVpjfycHHw/cz5M+O48IiIqQ7n/dfD09MShQ4ee+fiRI0fg7e2tlVBE/6t+61Z4dfQHAIC9c79E6sNYiRMREVFFVe5yY29vX+b9bf5WVFQEOzs7rYQi+ier6tXw7qIgyBUK/PnzAUTuf3bJJiIiKne5iY2NRbNmzZ75eIsWLRAfH6+VUET/NHDuDFSzt0NS9AP8MH+Z1HGIiKiCK3e5OXjwIObOnQtzc/NSj1WpUgVBQUHYv3+/VsMRdXx3IJp27oDC/Hxs+3wGCnJzpY5EREQVXLnfCj5v3jz07dsXd+7cwerVq3H79m0AgIeHB8aOHQuFQoH58+frLChVPrWbuOONiWMBAD8vWYH4O/ckTkRERIag3OUmKSkJ7dq1w7p167Bw4ULIZDIAgBAChw8fxtixY5GUlKSzoFS5mFtZYuiX82BiaoorR39D+J4fpY5EREQGQqOb+P3111/o2bMnqlevjoYNG0Imk+Hu3bvIyMjQUTyqrPrPmgpbl9pIi4vHnjkLpY5DREQG5IXuUJyRkYHz589rOwsRAMCnT094vf4qiouKsHPKbORmlb63EhER0bPwLmhUodRyrYu3AicBAA6t/hoxl69KnIiIiAwNyw1VGCbm5hj65VyYW1rgTvg5/LZ5h9SRiIjIALHcUIXR6/NP4eTeCNmpadgVGAQhhNSRiIjIALHcUIXQ3N8P7Qf1AwDsCgxCdmqaxImIiMhQsdyQ5GwcHTAgeBoA4PimbbgTfk7iREREZMhYbkhSchMF3l0SDEtrazy4fA2/rv5a6khERGTgWG5IUq+N+RD1WjZHblY2dkydhZKiYqkjERGRgWO5Ick0auuDriOGAgD2BC1CWiw/eJWIiF4eyw1JompNGwxeMAtyuRxn9vyIK0eOSx2JiIiMBMsN6Z1MJsM782bB2s4W8Xej8POSFVJHIiIiI8JyQ3rnN3wwPDq0RUFuHrZPnomi/HypIxERkRFhuSG9qtOsCV4fNxoA8NPi5UiMipY4ERERGRuWG9KbKlWtMPTLYChMTXDpUBjO7v1F6khERGSEWG5Ib96e/QVq1nZG6qM4/CdokdRxiIjISLHckF606fsmWgZ0Q3FhEXZMmYm8x0+kjkREREaK5YZ0zr5+PfT5YiIA4NdV6/HX1RsSJyIiImPGckM6ZWJujqFL58HMogpun47A71t3SR2JiIiMHMsN6VTvyePg2KgBslJSsWt6MIQQUkciIiIjx3JDOtOiexe0G9gXJSUl2BUYhMep6VJHIiKiSoDlhnTCxskBA+YEAgCOb9qOuxF/SpyIiIgqC5Yb0jq5iQLvLgmGhbUSMZeu4vDajVJHIiKiSoTlhrSuxyejUM+zOXKysrBjyiyUFBVLHYmIiCoRlhvSKjffV9B1xDAAwJ5ZC5AenyBxIiIiqmxYbkhrlDVrYPDC2QCA09/uxdVjJyRORERElRHLDWmFTCbD4EVzoKxZA3G37+KXL1dKHYmIiCoplhvSiq4jhsGtrQ/yc3KxffJMFBUUSB2JiIgqKZYbemn1WrbAa2NHAgB+XLgMSdEPJE5ERESVGcsNvRQLa2u8uyQIChMTRO4/hD9/OiB1JCIiquRYbuilDAyeBhtHByTH/IW9c7+UOg4RERHLDb24DoP7o7m/H4oKCrB9ykzk5+RIHYmIiIjlhl6Mc2M3vDnpUwDAvmWrEXvzjsSJiIiInmK5IY2ZW1pi6JK5MDEzw7XjJ3Bq13+kjkRERKTCckMa6z9rCuzquSA9PgHfzlwgdRwiIiI1FaLcjBkzBtHR0cjNzUVERAR8fHyeue/IkSNx8uRJpKWlIS0tDUePHn3u/qRdr/R5A149X0NxURF2TJmN3KwsqSMRERGpkbzcDBgwACEhIQgKCoKXlxcuX76Mw4cPw87Orsz9O3fujN27d6NLly7w9fXFw4cPceTIETg5Oek5eeVjX78e3po2CQBwaPVGxFy6InEiIiKisgkpV0REhFi1apXqa5lMJh49eiSmTp1aruPlcrnIzMwUQ4cOLdf+SqVSCCGEUqnU+s+iy3NLvUyrmIvPf9ghll0NF6M2fCVkMpmkeYx51hVtcdactTEuztrwZq3JeSR95sbU1BTe3t4ICwtTbRNCICwsDL6+vuU6h6WlJUxNTZGWlqarmASgz9QJcGzUAFkpqdg1LQhCCKkjERERlclEym9ua2sLExMTJCYmqm1PTEyEh4dHuc6xePFixMXFqRWkfzIzM4O5ubnqa6VSqfarNuny3FJq1q0z2vbvDVFSgh+DFkNWUCT5z2iss66IOGv94az1h7PWH23NWpPjJS03L2vq1KkYNGgQOnfujPz8/DL3CQwMxJw5c0ptj42N1VkuXZ5b39Lzc7Ej6ioKS0rQ1r4OJv12UupIaoxp1hUdZ60/nLX+cNb6o89ZS1puUlJSUFRUBHt7e7Xt9vb2SEhIeO6xkyZNwhdffIFu3brh6tWrz9xv4cKFCAkJUX2tVCoRGxsLZ2dnZGdnv9wP8D90eW4pmJiZYsTXK+Do1hAPLl1F8KcBKCkukToWAOObdUXGWesPZ60/nLX+aGvWf5+nvCS90CgiIkKsXLlS9bVMJhMPHz587gXFkydPFhkZGaJNmzaSXdik73NLsd6aNkksuxougk/+Kqxr2Umex5hnXZEXZ81ZG+PirA1v1pqcR/KXpUJCQhAaGorz58/j3LlzGD9+PKysrLBlyxYAQGhoKGJjYzFt2jQAwJQpUxAcHIzBgwcjJiZG9azP48eP8eTJE8l+DmPTvFtndHinPwBg17QgZCUlS5yIiIiofCQvN3v27IGdnR2Cg4Ph4OCAS5cuISAgAElJSQAAFxcXlJT896WQjz/+GObm5ti7d6/aeebMmYOgoCC9ZjdWNZwdMTDoaZk8vnk7bp2KkDgRERFR+UlebgBgzZo1WLNmTZmPdenSRe1rV1dXfUSqtBQmJnh3yVxYWCsRc+kqfl21QepIREREGpH8DsVUsfScMAZ1WzRFTmYWtk+eiZKiYqkjERERaYTlhlSadukIv2HvAAB2T5+LjITEfzmCiIio4mG5IQCAjZMDBs2bAQD4PXQXbpw4JXEiIiKiF8NyQ1CYmGDol/NgaW2NB5ev4eBX66SORERE9MJYbui/19lkPb3OprioSOpIREREL4zlppL753U2386Yh/T4598ZmoiIqKJjuanEajg7qq6zObH9W1z/7Q+JExEREb08lptKSu06myvXcSCk7PsMERERGRqWm0rqjYmfwKV5k6f3s/l8Bq+zISIio8FyUwk19/dDp6EDATy9nw2vsyEiImPCclPJ1KjthIHB0wEAv23ZyfvZEBGR0WG5qUQUpqYYtnQeLKyViL54BQdX8n42RERkfFhuKpFek8ehTtPGeJKegR383CgiIjJSLDeVRMvX/NHhnf4AgF3TgpCRmCRxIiIiIt1guakE7Oq54O2gQABA2NdbcetUhMSJiIiIdIflxsiZVjHH8JAFqGJlhXvnInF47TdSRyIiItIplhsj13f653Bs1ABZKanYMXU2Sop5nQ0RERk3lhsj5tOnJ17p8wZKiouxY/JMZKekSh2JiIhI51hujJSjW0P0mz4ZAHBo9UZEnb8ocSIiIiL9YLkxQlWqWmF4yAKYVjHHjZOncXzTNqkjERER6Q3LjREaOHcG7OrWQVpsPHYFBkMIIXUkIiIivWG5MTKdhg1Ci26dUVRYiG2TpiM3K0vqSERERHrFcmNEXFu1wBsTxgIAfl78FR5evylxIiIiIv1juTESVWvaYOjSeVCYmODCgcM4890PUkciIiKSBMuNEZArFBi6ZC6q1bJDQlQ0/hO0WOpIREREkmG5MQI9Ph2Fhq94I+/JE4ROCERBbq7UkYiIiCTDcmPgmnXthK4jhgEAvpu1AEnRDyROREREJC2WGwNm61Ibg+bNBACc2LYbV44clzgRERGR9FhuDJSZRRW899UiWCir4n7kJexfvkbqSERERBUCy42B6jdziuoDMbdPnomSIn4gJhEREcByY5DaD+qH1m/2QHFREbZPnoms5BSpIxEREVUYLDcGpp5nc/SeMh4AcGD5WtznB2ISERGpYbkxIMqaNTAsZD4Upia4dPgYTmzbLXUkIiKiCoflxkDITRQYunSe6kZ9e2YtkDoSERFRhcRyYyDemDAWDVq3Qt7jJ9g6/gvk5+RIHYmIiKhCYrkxAC0DusFv2DsAgN3T5yI55i+JExEREVVcLDcVnKNbAwwImgYAOL5pG64dPyFxIiIiooqN5aYCs7BW4r2vFsHc0gK3z5zFwZUbpI5ERERU4bHcVFAyuRxDFs2BbZ3aSH0Uhx1TZkGUlEgdi4iIqMJjuamgXhszEo07tkNhXj5CJwQiJzNL6khEREQGgeWmAmrWtRO6f/Q+AGBP0ELE3rojcSIiIiLDwXJTwdRyrYt35s8CAJzc8R0u7D8scSIiIiLDwnJTgVSpaoX3VyxGlapWiDp/EfuWrZI6EhERkcFhuakgZDIZhiwKQi3XukiPT8C2SdP5Sd9EREQvgOWmgnht7Ido4tcehXn52Dr+CzxOS5c6EhERkUFiuakAmnfrrLqA+D9Bi/Doxm2JExERERkulhuJOTSsj3fmzwQAnNj+LSL3H5I4ERERkWFjuZGQZTVrvL9yMcwtLXE34jz2L1stdSQiIiKDx3IjEblCgaFL56nuQLx98gyUFPMCYiIiopfFciORNyd9Cre2PsjPycHmcVPwJCNT6khERERGgeVGAj59eqLT0IEAgN3TgpFwN0riRERERMaD5UbP6no2Q/+ZUwAAh9d+g6vHTkiciIiIyLiw3OhRNXs7vLd8IUzMzHAl7HccXb9Z6khERERGh+VGT0yrmOP9lUtgbWeLuDv3sHtaMIQQUsciIiIyOiw3ejJo3kzUaeKBx2np2PzpZBTk5kodiYiIyCix3OhB99EfoOVr/igqLEToxGlIj0uQOhIREZHRYrnRsebdOiNg7IcAgL1zv8T9yEvSBiIiIjJyLDc65OzhhnfmzwLw9KMVzv24T+JERERExo/lRkeUtjXxwaolMLe0wK1TEfxoBSIiIj2pEOVmzJgxiI6ORm5uLiIiIuDj4/Pc/fv374+bN28iNzcXV65cQY8ePfSUtHxMzMzwwcolqO5gj8T7Mdg+ZSY/WoGIiEhPJC83AwYMQEhICIKCguDl5YXLly/j8OHDsLOzK3N/X19f7N69G5s2bUKrVq3w008/4aeffkLTpk31nLxsQgj0mfE5XJo3wZP0DGwa+znysh9LHYuIiKhSEVKuiIgIsWrVKtXXMplMPHr0SEydOrXM/b/99luxb98+tW3h4eFi3bp15fp+SqVSCCGEUqnU+s+iVCrF6YS/xLKr4WLxhZOivndLSWdrzEuXf45cnDVnbfyLsza8WWtyHhNIyNTUFN7e3li4cKFqmxACYWFh8PX1LfMYX19fhISEqG07fPgw+vTpU+b+ZmZmMDc3V32tVCrVftUmnzd7ICI5FgCwf8kKJN+J0sn3Id3+OZI6zlp/OGv94az1R1uz1uR4ScuNra0tTExMkJiYqLY9MTERHh4eZR7j4OBQ5v4ODg5l7h8YGIg5c+aU2h4bG/tioZ8hLicb/4m+gWIh0NrWERN3f6/V81PZtP3nSM/GWesPZ60/nLX+6HPWkpYbfVi4cKHaMz1KpRKxsbFwdnZGdna21r6PU2M3DF4yF5716mOwTwdkZWZq7dxUmq7+HKk0zlp/OGv94az1R1uz/vs85SFpuUlJSUFRURHs7e3Vttvb2yMhoey7+CYkJGi0f0FBAQoKCkptz87O1upf6NvnIvH1B2Nx/85dZGVm8n8seqLtP0d6Ns5afzhr/eGs9Uefs5b03VKFhYWIjIyEv7+/aptMJoO/vz/Cw8PLPCY8PFxtfwDo3r37M/fXp6zkFJgpFFLHICIiqtQkf1kqJCQEoaGhOH/+PM6dO4fx48fDysoKW7ZsAQCEhoYiNjYW06ZNAwCsWLECJ06cwMSJE3HgwAEMGjQIrVu3xqhRo6T8MYiIiKiCkLzc7NmzB3Z2dggODoaDgwMuXbqEgIAAJCUlAQBcXFxQUlKi2j88PByDBw/GvHnzsGDBAty9exd9+vTB9evXpfoRiIiIqIKR/D3w+ly6vs8N75tg+H+OXJw1Z238i7M2vFlrch7J71BMREREpE0sN0RERGRUWG6IiIjIqLDcEBERkVFhuSEiIiKjwnJDRERERoXlhoiIiIwKyw0REREZFZYbIiIiMiosN0RERGRUJP9sKakolUqdnVMX5yZ1nLX+cNb6w1nrD2etP9qatSbHy/D0cxgqDScnJ8TGxkodg4iIiF6As7Mz4uLinrtPpSs3wNOCk52drfXzKpVKxMbGwtnZWSfnp//irPWHs9Yfzlp/OGv90easlUrlvxYboJK+LFWewbyM7Oxs/o9FTzhr/eGs9Yez1h/OWn+0MevyHs8LiomIiMiosNwQERGRUWG50aL8/HzMmTMH+fn5Ukcxepy1/nDW+sNZ6w9nrT9SzLpSXlBMRERExovP3BAREZFRYbkhIiIio8JyQ0REREaF5YaIiIiMCsuNlowZMwbR0dHIzc1FREQEfHx8pI5k8L744gucO3cOWVlZSExMxI8//gg3Nze1fczNzbF69WqkpKQgOzsb33//PWrVqiVRYuMxdepUCCGwfPly1TbOWnucnJywfft2pKSkICcnB1euXIG3t7faPkFBQYiLi0NOTg6OHj2Khg0bSpTWcMnlcgQHB+P+/fvIycnBvXv3MGPGjFL7cdYvpmPHjvjll18QGxsLIQR69+5dap9/m62NjQ127NiBzMxMpKen45tvvoGVlZVW8gmul1sDBgwQeXl54r333hONGzcWGzZsEGlpacLOzk7ybIa8fv31VzF8+HDRpEkT0aJFC7F//34RExMjLC0tVfusXbtWPHjwQHTp0kV4eXmJM2fOiFOnTkme3ZBX69atxf3798WlS5fE8uXLOWstr+rVq4vo6GixefNm4ePjI+rVqye6d+8u6tevr9pnypQpIj09XfTq1Us0b95c/PTTTyIqKkqYm5tLnt+QVmBgoEhOThavv/66qFu3rujXr5/IysoSn376KWethRUQECDmzp0r+vTpI4QQonfv3mqPl2e2Bw8eFBcvXhSvvPKKaN++vbhz547YuXOnNvJJPyBDXxEREWLVqlWqr2UymXj06JGYOnWq5NmMadna2gohhOjYsaMAIKytrUV+fr7o16+fah93d3chhBBt2rSRPK8hLisrK3H79m3h7+8vfvvtN1W54ay1txYuXChOnjz53H3i4uLEpEmTVF9bW1uL3NxcMXDgQMnzG9Lat2+f+Oabb9S2ff/992L79u2ctZZXWeXm32br4eEhhBDC29tbtc9rr70miouLhaOj40vl4ctSL8nU1BTe3t4ICwtTbRNCICwsDL6+vhImMz7VqlUDAKSlpQEAvL29YWZmpjb727dv48GDB5z9C1qzZg0OHDiAY8eOqW3nrLWnV69eOH/+PPbs2YPExERcuHABI0eOVD3u6uoKR0dHtVlnZWXh7NmznLWGzpw5A39/fzRq1AgA0KJFC3To0AG//vorAM5al8ozW19fX6SnpyMyMlK1T1hYGEpKStCmTZuX+v6V8oMztcnW1hYmJiZITExU256YmAgPDw+JUhkfmUyGr776CqdOncL169cBAA4ODsjPz0dmZqbavomJiXBwcJAipkEbOHAgvLy8yrxejLPWnvr16+Pjjz9GSEgIFixYAB8fH6xcuRIFBQXYtm2bap5l/TeFs9bMokWLYG1tjVu3bqG4uBgKhQLTp0/Hrl27AICz1qHyzNbBwQFJSUlqjxcXFyMtLe2l589yQwZhzZo1aNasGTp06CB1FKNUu3ZtrFixAt27d+ft6HVMLpfj/PnzmD59OgDg0qVLaNasGUaPHo1t27ZJnM64DBgwAEOGDMHgwYNx/fp1tGzZEl999RXi4uI4ayPHl6VeUkpKCoqKimBvb6+23d7eHgkJCRKlMi6rVq3CG2+8gS5duiA2Nla1PSEhAebm5qqXq/7G2WvO29sb9vb2uHDhAgoLC1FYWIjOnTtj3LhxKCwsRGJiImetJfHx8bhx44batps3b8LFxQUAVPPkf1Ne3pdffolFixbhu+++w7Vr17Bjxw4sX74cgYGBADhrXSrPbBMSEkq941KhUKBGjRovPX+Wm5dUWFiIyMhI+Pv7q7bJZDL4+/sjPDxcwmTGYdWqVXjrrbfQtWtXxMTEqD0WGRmJgoICtdm7ubmhbt26nL2Gjh07hmbNmqFly5aq9eeff2Lnzp1o2bIlzp8/z1lryenTp+Hu7q62zc3NDQ8ePAAAREdHIz4+Xm3WSqUSbdq04aw1ZGlpiZKSErVtxcXFkMuf/tPHWetOeWYbHh4OGxsbeHl5qfbp2rUr5HI5zp49+9IZJL/K2tDXgAEDRG5urhg2bJjw8PAQ69evF2lpaaJWrVqSZzPktWbNGpGeni46deok7O3tVatKlSqqfdauXStiYmJE586dhZeXlzh9+rQ4ffq05NmNYf3z3VKctfZW69atRUFBgQgMDBQNGjQQ77zzjnj8+LEYPHiwap8pU6aItLQ08eabb4pmzZqJH3/8kW9PfoG1ZcsW8fDhQ9Vbwfv06SOSkpLEokWLOGstLCsrK+Hp6Sk8PT2FEEKMHz9eeHp6ijp16pR7tgcPHhSRkZHCx8dHtGvXTty+fZtvBa9Ia+zYsSImJkbk5eWJiIgI8corr0ieydDXswwfPly1j7m5uVi9erVITU0Vjx8/Fnv37hX29vaSZzeG9b/lhrPW3urZs6e4cuWKyM3NFTdu3BAjR44stU9QUJCIj48Xubm54ujRo6JRo0aS5za0VbVqVbF8+XIRExMjcnJyxL1798TcuXOFqakpZ62F5efnV+Z/o7ds2VLu2drY2IidO3eKrKwskZGRITZt2iSsrKxeOpvs/39DREREZBR4zQ0REREZFZYbIiIiMiosN0RERGRUWG6IiIjIqLDcEBERkVFhuSEiIiKjwnJDRERERoXlhoiMjqmpKe7evQtfX1+po5Ty0Ucf4ZdffpE6BpFRY7khon9la2uLtWvX4sGDB8jLy0N8fDwOHTqEdu3aqfYRQqB3794Spvyv0aNHIzo6WqPPB3rrrbdw+PBhpKSkQAgBT0/PUvuYm5tj9erVSElJQXZ2Nr7//vtSH/xXp04d7N+/H0+ePEFiYiKWLFkChUKhenzz5s3w8vLiJ9wT6RDLDRH9q71796JVq1YYPnw43Nzc0KtXL/z++++oWbOm1NHK9Mknn2DTpk0aHWNlZYVTp05h6tSpz9xn+fLlePPNN/H222/Dz88PTk5O+OGHH1SPy+VyHDhwAGZmZmjXrh2GDx+O9957D8HBwap9CgsLsWvXLowbN07zH4yIyk3yz6fg4uKquKtatWpCCCE6der0zH2io6PVPlsmOjpa9VivXr1EZGSkyM3NFVFRUWLWrFlCoVCoHhdCiNGjR4uDBw+KnJwcERUVJfr166d63NTUVKxatUrExcWJ3NxcERMTI7744otnZvH29hZFRUWiatWqqm1Dhw4V2dnZomHDhqpta9asETdv3hQWFhZqx9etW1cIIYSnp6fadmtra5Gfn6+Wzd3dXQghRJs2bQQAERAQIIqKitQ+NPejjz4SGRkZap9n1LFjR5GXl6f2IbBcXFxaXZIH4OLiqsBLoVCIrKwsERISIszMzMrcx9bWVvWhpvb29sLW1lYAEB06dBAZGRli2LBhwtXVVXTr1k3cv39fzJo1S3WsEEIkJyeLESNGiEaNGong4GBRWFgoPDw8BAAxadIk8eDBA9GhQwfh4uIi2rdvLwYNGvTMvOPHjxc3btwotf27774TZ8+eFQqFQrz++usiPz9feHl5ldrvWeWmS5cuQgghqlWrprY9JiZGjB8/XgBPPyTw4sWLao/Xq1dPCCFEy5YtVdssLCxEUVGR8PPzk/zPl4vLSJfkAbi4uCr46tu3r0hNTRU5OTni1KlTYv78+aJ58+Zq+wghRO/evdW2HT16tNSzLEOGDBGxsbFqx61du1Ztn/DwcLFmzRoBQKxYsUKEhYWVO+vy5cvL3L969erir7/+EmvWrBHx8fEiMDCwzOOfVW7eeecdkZeXV2r/s2fPikWLFgkAYsOGDeLQoUNqj1tYWAghhAgICFDbnpqaKoYNGyb5ny0XlzEuXnNDRP/qhx9+gJOTE3r16oVDhw6hc+fOuHDhAoYPH/7c4zw9PTFr1ixkZ2er1saNG+Hk5AQLCwvVfv974W94eDgaN24MANi6dStatmyJ27dvY8WKFejevftzv6eFhQXy8vJKbc/IyMCIESMwZswYREVFYdGiReX98XUiNzcXlpaWkmYgMlYsN0RULvn5+QgLC8O8efPQvn17bN26FUFBQc89pmrVqpg9ezZatmypWs2bN0fDhg3LLCBluXjxIlxdXTFz5kxYWFhgz549+M9//vPM/VNSUmBjY1PmY506dUJRUREcHR1hZWVVru//t4SEBJibm6NatWpq2+3t7ZGQkKDax97evtTjfz/2TzVq1EBycrJGGYio/CR/+oiLi8vw1oQJE0RycrLq6/z8fNG3b1+1fU6dOiW++eab555HCKF6CervdebMmVLb/l6vvvqqEEIIGxubMh/v16+fSE1NLbXd19dXFBQUiICAAHH58mWxdevWMo//twuK//kzurm5lXlBsZ2dnWqfDz/8UGRkZKhdr1S/fn0hhBD169eX/M+Ri8tIl+QBuLi4KvCqUaOGOHbsmBgyZIho3ry5qFevnujfv7+Ij49XKy63b98Wa9asEfb29qJ69eoCeFpECgoKxKxZs0STJk2Eh4eHGDhwoJg7d67qOCGESEpKEu+//75o1KiRmDNnjigqKhKNGzcWwNMSNWjQIOHu7i4aNWokNm7cKOLi4oRMJntm3vz8fNG0aVPVtqpVq4p79+6JpUuXCgCiWbNmIjc3V+2dTzY2NsLT01P06NFDCCHEgAEDhKenp7C3t1fts3btWhETEyM6d+4svLy8xOnTp8Xp06dVj8vlcnHlyhVx6NAh0aJFC/Hqq6+KxMREMX/+fLWMw4cPF/fu3ZP8z5aLy4iX5AG4uLgq8DIzMxMLFiwQ58+fF+np6eLx48fi5s2bIjg4WO2tzG+88Ya4c+eOKCgoUHsr+KuvvipOnTolnjx5IjIyMkRERIQYOXKk6nEhhPj444/F4cOHRW5urrh//754++23VY+PHDlSXLhwQWRnZ4uMjAxx9OhRtXcelbW+/fZbsWDBAtXXmzZtEpcvX1Z79mTChAkiJSVFODk5CeBp4SjL7NmzVceYm5uL1atXi9TUVPH48WOxd+9etfIDQLi4uIgDBw6IJ0+eiKSkJPHll1+qvfUdgDh06JCYOnWq5H+2XFxGvCQPwMXFVYlXWe+yetnVvHlzkZCQIKysrCT/+f53NWnSRCQkJAhra2vJs3BxGeviBcVEZHSuXr2KqVOnwtXVVeoopTg6OmLYsGHIysqSOgqR0ZLhacshIpKEEAJ9+vTBzz//LHUUIjISLDdERERkVPiyFBERERkVlhsiIiIyKiw3REREZFRYboiIiMiosNwQERGRUWG5ISIiIqPCckNERERGheWGiIiIjArLDRERERmV/wOciZDfhwtZoQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(net.similarities, label=\"Cosine similarity of weights between heads\")\n",
    "plt.xlabel(\"Steps (x100)\")\n",
    "plt.ylabel(\"Cosine similarity\")\n",
    "\n",
    "plt.grid()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0152,  0.0601,  0.0390,  ...,  0.1106, -0.1470, -0.0648],\n",
       "        [ 0.0235, -0.0084, -0.0743,  ...,  0.1758,  0.1622,  0.1090],\n",
       "        [-0.0309, -0.0267,  0.0058,  ...,  0.0113, -0.0673, -0.0841],\n",
       "        ...,\n",
       "        [ 0.0373, -0.2460,  0.0477,  ..., -0.1265,  0.0965,  0.0238],\n",
       "        [-0.0761,  0.1184, -0.0040,  ...,  0.0410, -0.1313,  0.0612],\n",
       "        [ 0.0417,  0.0181, -0.0737,  ..., -0.1700, -0.0005,  0.0184]])"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.head_2_1.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.0158,  0.0927,  0.0358,  ...,  0.1278, -0.1246, -0.0654],\n",
       "        [ 0.0090, -0.0662, -0.0603,  ...,  0.1438,  0.1862,  0.0725],\n",
       "        [-0.0677, -0.0098,  0.0134,  ...,  0.0088, -0.0573, -0.0782],\n",
       "        ...,\n",
       "        [ 0.0391, -0.2793,  0.0162,  ..., -0.1219,  0.1442,  0.0538],\n",
       "        [-0.0944,  0.1313,  0.0364,  ...,  0.0551, -0.1656,  0.0890],\n",
       "        [ 0.0621,  0.0341, -0.0559,  ..., -0.2171, -0.0171,  0.0220]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.head_1_1.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.9676, grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.cosine_similarity(net.head_1_1.weight, net.head_2_1.weight).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We set \\sigma = 1 based on the cited papers 18 and 19 in ReConPatch\n",
    "class ContextualSimilarity(nn.Module):\n",
    "    def __init__(self, k=5, alpha=0.5):\n",
    "        super(ContextualSimilarity, self).__init__()\n",
    "        self.k = k\n",
    "        self.alpha = alpha\n",
    "\n",
    "    def forward(self, z):\n",
    "        distances = torch.cdist(z, z)\n",
    "        topk_dist = -torch.topk(-distances, self.k)[0][:, -1]\n",
    "        filtering = (distances <= topk_dist.unsqueeze(-1)).float()\n",
    "        similarity = torch.matmul(filtering, filtering.transpose(0, 1)) / torch.sum(\n",
    "            filtering, dim=-1, keepdim=True\n",
    "        )\n",
    "        R = filtering * filtering.transpose(0, 1)\n",
    "        similarity = torch.matmul(similarity, R.transpose(0, 1)) / torch.sum(\n",
    "            R, dim=-1, keepdim=True\n",
    "        )\n",
    "        return self.alpha * (similarity + similarity.transpose(0, 1))\n",
    "\n",
    "\n",
    "class PairwiseSimilarity(nn.Module):\n",
    "    def __init__(self, sigma=1.0):\n",
    "        super(PairwiseSimilarity, self).__init__()\n",
    "        self.sigma = sigma\n",
    "\n",
    "    def forward(self, z):\n",
    "        return torch.exp(-(torch.cdist(z, z) / self.sigma))\n",
    "\n",
    "\n",
    "def relaxedContrastiveLoss(w, delta, m):\n",
    "    return torch.mean(\n",
    "        w * delta**2 + (1 - w) * torch.max(m - delta, torch.tensor(0.0)) ** 2\n",
    "    )\n",
    "\n",
    "\n",
    "class RelaxedContrastiveLoss(nn.Module):\n",
    "    def __init__(self, margin):\n",
    "        super(RelaxedContrastiveLoss, self).__init__()\n",
    "        self.margin = margin\n",
    "\n",
    "    def forward(self, w, delta):\n",
    "        return torch.mean(\n",
    "            w * delta**2\n",
    "            + (1 - w) * torch.max(self.margin - delta, torch.tensor(0.0)) ** 2\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReConPatch(lightning.LightningModule):\n",
    "\n",
    "    def __init__(\n",
    "        self, input_dim=1536, emb_dim=512, proj_dim=1024, alpha=0.5, margin=0.1\n",
    "    ):\n",
    "\n",
    "        super(ReConPatch, self).__init__()\n",
    "\n",
    "        self.emb_dim = emb_dim\n",
    "        self.proj_dim = proj_dim\n",
    "        self.alpha = alpha\n",
    "        self.margin = margin\n",
    "\n",
    "        self.fmap_buf = []\n",
    "        self.memory_bank = []\n",
    "\n",
    "        self.wr_model = torch.hub.load(\n",
    "            \"pytorch/vision:v0.13.0\", \"wide_resnet50_2\", pretrained=True\n",
    "        ).eval()\n",
    "\n",
    "        # freeze the model\n",
    "        for param in self.wr_model.parameters():\n",
    "            param.requires_grad = False\n",
    "\n",
    "        def hook(module, input, output):\n",
    "            self.fmap_buf.append(output)\n",
    "\n",
    "        self.wr_model.layer2.register_forward_hook(hook)\n",
    "        self.wr_model.layer3.register_forward_hook(hook)\n",
    "\n",
    "        # Online head\n",
    "\n",
    "        self.repr_layer = nn.Linear(input_dim, emb_dim)\n",
    "        self.proj_layer = nn.Linear(emb_dim, proj_dim)\n",
    "\n",
    "        # Target head\n",
    "\n",
    "        self.target_repr_layer = EMA_linear(input_dim, emb_dim, steps_to_update=1)\n",
    "        self.target_proj_layer = EMA_linear(emb_dim, proj_dim, steps_to_update=1)\n",
    "\n",
    "        # Filters\n",
    "\n",
    "        self.blur = nn.AvgPool2d(kernel_size=3, stride=1)\n",
    "\n",
    "        self.activ = nn.LeakyReLU(negative_slope=1e-3)\n",
    "\n",
    "        self.apply(self._init_weights)\n",
    "\n",
    "        # step to have similar weights\n",
    "        self.target_proj_layer.update(self.proj_layer.weight, self.proj_layer.bias)\n",
    "        self.target_repr_layer.update(self.repr_layer.weight, self.repr_layer.bias)\n",
    "\n",
    "        self.pairwise_sim = PairwiseSimilarity()\n",
    "        self.contextual_sim = ContextualSimilarity()\n",
    "        self.criterion = RelaxedContrastiveLoss(self.margin)\n",
    "\n",
    "    def forward(self, X):\n",
    "        \"\"\"forward\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : torch.Tensor\n",
    "            dimensions B x C x H x W\n",
    "        \"\"\"\n",
    "\n",
    "        # empty out fmap buffer\n",
    "        self.fmap_buf = []\n",
    "\n",
    "        # trigger forward pass (and hook)\n",
    "        _ = self.wr_model(X)\n",
    "\n",
    "        dims = (\n",
    "            int(max([map.shape[-2] for map in self.fmap_buf])),\n",
    "            int(max([map.shape[-1] for map in self.fmap_buf])),\n",
    "        )\n",
    "\n",
    "        pool = nn.AdaptiveAvgPool2d(dims)\n",
    "        preproc = lambda t: pool(self.blur(t))\n",
    "\n",
    "        feat_stacks = torch.cat([preproc(map) for map in self.fmap_buf], dim=1)\n",
    "        feat_stack_reshape = feat_stacks.reshape(feat_stacks.shape[1], -1).T\n",
    "\n",
    "        # Online head\n",
    "\n",
    "        repr = self.activ(self.repr_layer(feat_stack_reshape))\n",
    "        proj = self.proj_layer(repr)\n",
    "\n",
    "        p_sim = self.pairwise_sim(proj)\n",
    "        c_sim = self.contextual_sim(proj)\n",
    "\n",
    "        w = self.alpha * p_sim + (1 - self.alpha) * c_sim\n",
    "        # Target head\n",
    "\n",
    "        target_repr = self.target_repr_layer(feat_stack_reshape)\n",
    "        target_proj = self.target_proj_layer(target_repr)\n",
    "\n",
    "        pairwise = torch.cdist(target_proj, target_proj)\n",
    "\n",
    "        EPS = 1e-9\n",
    "\n",
    "        d = torch.sqrt(pairwise + EPS)\n",
    "        delta = d / d.mean()\n",
    "\n",
    "        return w, delta\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        x, y = batch\n",
    "        w, delta = self(x)\n",
    "\n",
    "        loss = self.criterion(w, delta)\n",
    "        self.log(\"train/loss\", loss)\n",
    "        return loss\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        x, y = batch\n",
    "        w, delta = self(x)\n",
    "\n",
    "        loss = self.criterion(w, delta)\n",
    "        self.log(\"val/loss\", loss)\n",
    "        return loss\n",
    "\n",
    "    def test_step(self, batch, batch_idx):\n",
    "        x, y = batch\n",
    "        w, delta = self(x)\n",
    "\n",
    "        loss = self.criterion(w, delta)\n",
    "        self.log(\"test/loss\", loss)\n",
    "        return loss\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        opt = torch.optim.AdamP(\n",
    "            self.parameters(), lr=1e-6, betas=(0.9, 0.999), weight_decay=1e-2\n",
    "        )\n",
    "        sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(\n",
    "            opt, T_0=10, T_mult=2\n",
    "        )\n",
    "\n",
    "        return [opt], [sched]\n",
    "\n",
    "    def save_bank(self, path):\n",
    "        torch.save(self.memory_bank, path)\n",
    "\n",
    "    def load_bank(self, path):\n",
    "        self.memory_bank = torch.load(path)\n",
    "\n",
    "    def _init_weights(self, m):\n",
    "        if isinstance(m, nn.Linear):\n",
    "            nn.init.kaiming_normal_(m.weight, a=1e-3)\n",
    "            nn.init.constant_(m.bias, 0)\n",
    "        elif isinstance(m, EMA_linear):\n",
    "            m.reset_parameters()\n",
    "\n",
    "    def on_after_backward(self):\n",
    "        # update target layers\n",
    "        self.target_proj_layer.update(self.proj_layer.weight, self.proj_layer.bias)\n",
    "        self.target_repr_layer.update(self.repr_layer.weight, self.repr_layer.bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".recon",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}