FinRL_compare_eRL_RLlib_stablebaseline.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "anaconda-cloud": {},
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.10"
    },
    "colab": {
      "name": "FinRL_compare_eRL_RLlib_stablebaseline.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AI4Finance-LLC/FinRL/blob/master/FinRL_compare_eRL_RLlib_stablebaseline.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "skemZ68ZZMdx"
      },
      "source": [
        "# FinRL: Compare ElegantRL, RLlib, and Stablebaselines3"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c3XX4MO9UxNx"
      },
      "source": [
        "## Install DRL libraries: FinRL, ElegantRL, RLlib"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VtxFMjnLDy3Q",
        "outputId": "8f492493-1d6b-41d8-ef8c-bbc04addbe24"
      },
      "source": [
        "## install elegantrl library\n",
        "!pip install git+https://github.com/AI4Finance-LLC/ElegantRL.git\n",
        "## install rllib/ray library\n",
        "!pip install ray[default]\n",
        "## install finrl library\n",
        "!pip install git+https://github.com/AI4Finance-LLC/FinRL-Library.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/AI4Finance-LLC/ElegantRL.git\n",
            "  Cloning https://github.com/AI4Finance-LLC/ElegantRL.git to /tmp/pip-req-build-4je3tzb_\n",
            "  Running command git clone -q https://github.com/AI4Finance-LLC/ElegantRL.git /tmp/pip-req-build-4je3tzb_\n",
            "Requirement already satisfied: gym in /usr/local/lib/python3.7/dist-packages (from elegantrl==0.3.1) (0.17.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from elegantrl==0.3.1) (3.2.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from elegantrl==0.3.1) (1.19.5)\n",
            "Collecting pybullet\n",
            "  Downloading pybullet-3.1.7.tar.gz (79.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 79.0 MB 26 kB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from elegantrl==0.3.1) (1.9.0+cu102)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from elegantrl==0.3.1) (4.1.2.30)\n",
            "Collecting box2d-py\n",
            "  Downloading box2d_py-2.3.8-cp37-cp37m-manylinux1_x86_64.whl (448 kB)\n",
            "\u001b[K     |████████████████████████████████| 448 kB 35.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym->elegantrl==0.3.1) (1.5.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym->elegantrl==0.3.1) (1.4.1)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym->elegantrl==0.3.1) (1.3.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym->elegantrl==0.3.1) (0.16.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->elegantrl==0.3.1) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->elegantrl==0.3.1) (1.3.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->elegantrl==0.3.1) (2.8.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->elegantrl==0.3.1) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10->matplotlib->elegantrl==0.3.1) (1.15.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->elegantrl==0.3.1) (3.7.4.3)\n",
            "Building wheels for collected packages: elegantrl, pybullet\n",
            "  Building wheel for elegantrl (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for elegantrl: filename=elegantrl-0.3.1-py3-none-any.whl size=65731 sha256=798e7bb61987d4d120b5ace18c196464cdb7d64f700a8b4edd5367761f89b959\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-9c3cyy5z/wheels/52/9a/b3/08c8a0b5be22a65da0132538c05e7e961b1253c90d6845e0c6\n",
            "  Building wheel for pybullet (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pybullet: filename=pybullet-3.1.7-cp37-cp37m-linux_x86_64.whl size=89750825 sha256=b706080493bb980942be324d481d7fc0af093ef495092d060a0dbc673eaa328d\n",
            "  Stored in directory: /root/.cache/pip/wheels/70/1c/62/86c8b68885c24123d87c5392d6678aa2b68a1796c8113e1aa6\n",
            "Successfully built elegantrl pybullet\n",
            "Installing collected packages: pybullet, box2d-py, elegantrl\n",
            "Successfully installed box2d-py-2.3.8 elegantrl-0.3.1 pybullet-3.1.7\n",
            "Collecting ray[default]\n",
            "  Downloading ray-1.5.2-cp37-cp37m-manylinux2014_x86_64.whl (51.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 51.0 MB 34 kB/s \n",
            "\u001b[?25hCollecting gpustat\n",
            "  Downloading gpustat-0.6.0.tar.gz (78 kB)\n",
            "\u001b[K     |████████████████████████████████| 78 kB 5.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: click>=7.0 in /usr/local/lib/python3.7/dist-packages (from ray[default]) (7.1.2)\n",
            "Collecting py-spy>=0.2.0\n",
            "  Downloading py_spy-0.3.8-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (3.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1 MB 34.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.16 in /usr/local/lib/python3.7/dist-packages (from ray[default]) (1.19.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from ray[default]) (3.13)\n",
            "Requirement already satisfied: protobuf>=3.15.3 in /usr/local/lib/python3.7/dist-packages (from ray[default]) (3.17.3)\n",
            "Requirement already satisfied: grpcio>=1.28.1 in /usr/local/lib/python3.7/dist-packages (from ray[default]) (1.39.0)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.7/dist-packages (from ray[default]) (2.6.0)\n",
            "Collecting colorama\n",
            "  Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
            "Collecting pydantic>=1.8\n",
            "  Downloading pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl (10.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 10.1 MB 29.2 MB/s \n",
            "\u001b[?25hCollecting aioredis<2\n",
            "  Downloading aioredis-1.3.1-py3-none-any.whl (65 kB)\n",
            "\u001b[K     |████████████████████████████████| 65 kB 3.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from ray[default]) (2.23.0)\n",
            "Collecting redis>=3.5.0\n",
            "  Downloading redis-3.5.3-py2.py3-none-any.whl (72 kB)\n",
            "\u001b[K     |████████████████████████████████| 72 kB 440 kB/s \n",
            "\u001b[?25hRequirement already satisfied: prometheus-client>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from ray[default]) (0.11.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from ray[default]) (3.0.12)\n",
            "Collecting opencensus\n",
            "  Downloading opencensus-0.7.13-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 59.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ray[default]) (1.0.2)\n",
            "Collecting aiohttp-cors\n",
            "  Downloading aiohttp_cors-0.7.0-py3-none-any.whl (27 kB)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3 MB 34.6 MB/s \n",
            "\u001b[?25hCollecting colorful\n",
            "  Downloading colorful-0.5.4-py2.py3-none-any.whl (201 kB)\n",
            "\u001b[K     |████████████████████████████████| 201 kB 61.5 MB/s \n",
            "\u001b[?25hCollecting async-timeout\n",
            "  Downloading async_timeout-3.0.1-py3-none-any.whl (8.2 kB)\n",
            "Collecting hiredis\n",
            "  Downloading hiredis-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl (85 kB)\n",
            "\u001b[K     |████████████████████████████████| 85 kB 3.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.5.2 in /usr/local/lib/python3.7/dist-packages (from grpcio>=1.28.1->ray[default]) (1.15.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from pydantic>=1.8->ray[default]) (3.7.4.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->ray[default]) (21.2.0)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.6.3-cp37-cp37m-manylinux2014_x86_64.whl (294 kB)\n",
            "\u001b[K     |████████████████████████████████| 294 kB 61.0 MB/s \n",
            "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-5.1.0-cp37-cp37m-manylinux2014_x86_64.whl (142 kB)\n",
            "\u001b[K     |████████████████████████████████| 142 kB 64.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: chardet<5.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->ray[default]) (3.0.4)\n",
            "Requirement already satisfied: idna>=2.0 in /usr/local/lib/python3.7/dist-packages (from yarl<2.0,>=1.0->aiohttp->ray[default]) (2.10)\n",
            "Requirement already satisfied: nvidia-ml-py3>=7.352.0 in /usr/local/lib/python3.7/dist-packages (from gpustat->ray[default]) (7.352.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from gpustat->ray[default]) (5.4.8)\n",
            "Collecting blessings>=1.6\n",
            "  Downloading blessings-1.7-py3-none-any.whl (18 kB)\n",
            "Requirement already satisfied: google-api-core<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from opencensus->ray[default]) (1.26.3)\n",
            "Collecting opencensus-context==0.1.2\n",
            "  Downloading opencensus_context-0.1.2-py2.py3-none-any.whl (4.4 kB)\n",
            "Requirement already satisfied: google-auth<2.0dev,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (1.34.0)\n",
            "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (57.4.0)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (1.53.0)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (21.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (2018.9)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (4.2.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (0.2.8)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=14.3->google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (2.4.7)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[default]) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->ray[default]) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->ray[default]) (1.24.3)\n",
            "Building wheels for collected packages: gpustat\n",
            "  Building wheel for gpustat (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gpustat: filename=gpustat-0.6.0-py3-none-any.whl size=12617 sha256=17b3e72ce345ee310a9a95b5d26dfaaff046769b26cc8f09ceab01be6c23a929\n",
            "  Stored in directory: /root/.cache/pip/wheels/e6/67/af/f1ad15974b8fd95f59a63dbf854483ebe5c7a46a93930798b8\n",
            "Successfully built gpustat\n",
            "Installing collected packages: multidict, yarl, async-timeout, opencensus-context, hiredis, blessings, aiohttp, redis, pydantic, py-spy, opencensus, gpustat, colorama, aioredis, aiohttp-cors, ray, colorful\n",
            "Successfully installed aiohttp-3.7.4.post0 aiohttp-cors-0.7.0 aioredis-1.3.1 async-timeout-3.0.1 blessings-1.7 colorama-0.4.4 colorful-0.5.4 gpustat-0.6.0 hiredis-2.0.0 multidict-5.1.0 opencensus-0.7.13 opencensus-context-0.1.2 py-spy-0.3.8 pydantic-1.8.2 ray-1.5.2 redis-3.5.3 yarl-1.6.3\n",
            "Collecting git+https://github.com/AI4Finance-LLC/FinRL-Library.git\n",
            "  Cloning https://github.com/AI4Finance-LLC/FinRL-Library.git to /tmp/pip-req-build-39b0pswi\n",
            "  Running command git clone -q https://github.com/AI4Finance-LLC/FinRL-Library.git /tmp/pip-req-build-39b0pswi\n",
            "Collecting pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2\n",
            "  Cloning https://github.com/quantopian/pyfolio.git to /tmp/pip-install-pm6trk34/pyfolio_369be7bd046442cab8cbe6cb12ca5f75\n",
            "  Running command git clone -q https://github.com/quantopian/pyfolio.git /tmp/pip-install-pm6trk34/pyfolio_369be7bd046442cab8cbe6cb12ca5f75\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.0) (1.19.5)\n",
            "Requirement already satisfied: pandas>=1.1.5 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.0) (1.1.5)\n",
            "Collecting stockstats\n",
            "  Downloading stockstats-0.3.2-py2.py3-none-any.whl (13 kB)\n",
            "Collecting yfinance\n",
            "  Downloading yfinance-0.1.63.tar.gz (26 kB)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.0) (3.2.2)\n",
            "Requirement already satisfied: scikit-learn>=0.21.0 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.0) (0.22.2.post1)\n",
            "Requirement already satisfied: gym>=0.17 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.0) (0.17.3)\n",
            "Collecting stable-baselines3[extra]\n",
            "  Downloading stable_baselines3-1.1.0-py3-none-any.whl (172 kB)\n",
            "\u001b[K     |████████████████████████████████| 172 kB 11.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pytest in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.0) (3.6.4)\n",
            "Requirement already satisfied: setuptools>=41.4.0 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.0) (57.4.0)\n",
            "Requirement already satisfied: wheel>=0.33.6 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.0) (0.37.0)\n",
            "Requirement already satisfied: ipython>=3.2.3 in /usr/local/lib/python3.7/dist-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (5.5.0)\n",
            "Requirement already satisfied: pytz>=2014.10 in /usr/local/lib/python3.7/dist-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (2018.9)\n",
            "Requirement already satisfied: scipy>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (1.4.1)\n",
            "Requirement already satisfied: seaborn>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (0.11.1)\n",
            "Collecting empyrical>=0.5.0\n",
            "  Downloading empyrical-0.5.5.tar.gz (52 kB)\n",
            "\u001b[K     |████████████████████████████████| 52 kB 1.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas-datareader>=0.2 in /usr/local/lib/python3.7/dist-packages (from empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (0.9.0)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17->finrl==0.3.0) (1.5.0)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17->finrl==0.3.0) (1.3.0)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (0.8.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (4.4.2)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (2.6.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (1.0.18)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (0.7.5)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (4.8.0)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (5.0.5)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->finrl==0.3.0) (2.8.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->finrl==0.3.0) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->finrl==0.3.0) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->finrl==0.3.0) (1.3.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10->matplotlib->finrl==0.3.0) (1.15.0)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (4.2.6)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (2.23.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (0.2.5)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym>=0.17->finrl==0.3.0) (0.16.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (1.24.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.0->finrl==0.3.0) (1.0.1)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from traitlets>=4.2->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (0.2.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.0) (0.7.0)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytest->finrl==0.3.0) (8.8.0)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.7/dist-packages (from pytest->finrl==0.3.0) (1.4.0)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.7/dist-packages (from pytest->finrl==0.3.0) (0.7.1)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from pytest->finrl==0.3.0) (21.2.0)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from pytest->finrl==0.3.0) (1.10.0)\n",
            "Requirement already satisfied: torch>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]->finrl==0.3.0) (1.9.0+cu102)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]->finrl==0.3.0) (7.1.2)\n",
            "Requirement already satisfied: tensorboard>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]->finrl==0.3.0) (2.6.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]->finrl==0.3.0) (5.4.8)\n",
            "Requirement already satisfied: atari-py~=0.2.0 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]->finrl==0.3.0) (0.2.9)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]->finrl==0.3.0) (4.1.2.30)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (0.6.1)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (3.17.3)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (1.0.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (0.4.5)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (0.12.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (3.3.4)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (1.34.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (1.8.0)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (1.39.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (4.2.2)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (4.6.3)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (3.1.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.4.0->stable-baselines3[extra]->finrl==0.3.0) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.0) (3.5.0)\n",
            "Collecting int-date>=0.1.7\n",
            "  Downloading int_date-0.1.8-py2.py3-none-any.whl (5.0 kB)\n",
            "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from yfinance->finrl==0.3.0) (0.0.9)\n",
            "Collecting lxml\n",
            "  Downloading lxml-4.6.3-cp37-cp37m-manylinux2014_x86_64.whl (6.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.3 MB 45.1 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: finrl, pyfolio, empyrical, yfinance\n",
            "  Building wheel for finrl (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for finrl: filename=finrl-0.3.0-py3-none-any.whl size=66464 sha256=59eb34f0b64a454a35f91153580d6c54b28ec37d53ef8ae385d884a2e90be13f\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-v0bcfi_q/wheels/17/ff/bd/1bc602a0352762b0b24041b88536d803ae343ed0a711fcf55e\n",
            "  Building wheel for pyfolio (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyfolio: filename=pyfolio-0.9.2+75.g4b901f6-py3-none-any.whl size=75775 sha256=5b3d7e42276dc0f1edc8c445e266a0dece4b249396a174cda7457f5c82d4eef3\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-v0bcfi_q/wheels/ef/09/e5/2c1bf37c050d22557c080deb1be986d06424627c04aeca19b9\n",
            "  Building wheel for empyrical (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for empyrical: filename=empyrical-0.5.5-py3-none-any.whl size=39777 sha256=41043f61bd029f3065fa1ecf0f337784de57d56289a306980ff7cbd7f9b39aac\n",
            "  Stored in directory: /root/.cache/pip/wheels/d9/91/4b/654fcff57477efcf149eaca236da2fce991526cbab431bf312\n",
            "  Building wheel for yfinance (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for yfinance: filename=yfinance-0.1.63-py2.py3-none-any.whl size=23918 sha256=c0fd0684257b0fbeefc91a5994aedd8761fa92139b9c249ab24ce006fb688cce\n",
            "  Stored in directory: /root/.cache/pip/wheels/fe/87/8b/7ec24486e001d3926537f5f7801f57a74d181be25b11157983\n",
            "Successfully built finrl pyfolio empyrical yfinance\n",
            "Installing collected packages: lxml, stable-baselines3, int-date, empyrical, yfinance, stockstats, pyfolio, finrl\n",
            "  Attempting uninstall: lxml\n",
            "    Found existing installation: lxml 4.2.6\n",
            "    Uninstalling lxml-4.2.6:\n",
            "      Successfully uninstalled lxml-4.2.6\n",
            "Successfully installed empyrical-0.5.5 finrl-0.3.0 int-date-0.1.8 lxml-4.6.3 pyfolio-0.9.2+75.g4b901f6 stable-baselines3-1.1.0 stockstats-0.3.2 yfinance-0.1.63\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cMCwv41eVVDJ"
      },
      "source": [
        "## Install additional packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P446vin0EGSC",
        "outputId": "037633da-6fe1-4eae-c968-01710741a531"
      },
      "source": [
        "!pip install trading_calendars\n",
        "!pip install alpaca_trade_api\n",
        "!pip install ccxt\n",
        "!pip install jqdatasdk\n",
        "!pip install wrds\n",
        "\n",
        "!pip install lz4\n",
        "!pip install ray[tune]\n",
        "!pip install tensorboardX\n",
        "!pip install gputil"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting trading_calendars\n",
            "  Downloading trading_calendars-2.1.1.tar.gz (108 kB)\n",
            "\u001b[?25l\r\u001b[K     |███                             | 10 kB 22.5 MB/s eta 0:00:01\r\u001b[K     |██████                          | 20 kB 26.2 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 30 kB 15.8 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 40 kB 10.8 MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 51 kB 9.6 MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 61 kB 8.1 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 71 kB 7.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 81 kB 8.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 92 kB 9.2 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▏ | 102 kB 8.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 108 kB 8.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from trading_calendars) (1.19.5)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from trading_calendars) (1.1.5)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from trading_calendars) (2.8.2)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from trading_calendars) (2018.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from trading_calendars) (1.15.0)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.7/dist-packages (from trading_calendars) (0.11.1)\n",
            "Building wheels for collected packages: trading-calendars\n",
            "  Building wheel for trading-calendars (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for trading-calendars: filename=trading_calendars-2.1.1-py3-none-any.whl size=140937 sha256=d5ce099fa375c9281de64b302454157371422b1607307e5add47b11f61ee4539\n",
            "  Stored in directory: /root/.cache/pip/wheels/62/9c/d1/46a21e1b99e064cba79b85e9f95e6a208ac5ba4c29ae5962ec\n",
            "Successfully built trading-calendars\n",
            "Installing collected packages: trading-calendars\n",
            "Successfully installed trading-calendars-2.1.1\n",
            "Collecting alpaca_trade_api\n",
            "  Downloading alpaca_trade_api-1.2.3-py3-none-any.whl (40 kB)\n",
            "\u001b[K     |████████████████████████████████| 40 kB 4.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: urllib3<2,>1.24 in /usr/local/lib/python3.7/dist-packages (from alpaca_trade_api) (1.24.3)\n",
            "Requirement already satisfied: msgpack==1.0.2 in /usr/local/lib/python3.7/dist-packages (from alpaca_trade_api) (1.0.2)\n",
            "Requirement already satisfied: requests<3,>2 in /usr/local/lib/python3.7/dist-packages (from alpaca_trade_api) (2.23.0)\n",
            "Collecting websocket-client<2,>=0.56.0\n",
            "  Downloading websocket_client-1.2.1-py2.py3-none-any.whl (52 kB)\n",
            "\u001b[K     |████████████████████████████████| 52 kB 1.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from alpaca_trade_api) (1.19.5)\n",
            "Collecting websockets<10,>=8.0\n",
            "  Downloading websockets-9.1-cp37-cp37m-manylinux2010_x86_64.whl (103 kB)\n",
            "\u001b[K     |████████████████████████████████| 103 kB 14.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from alpaca_trade_api) (1.1.5)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>2->alpaca_trade_api) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>2->alpaca_trade_api) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>2->alpaca_trade_api) (2.10)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->alpaca_trade_api) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->alpaca_trade_api) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->alpaca_trade_api) (1.15.0)\n",
            "Installing collected packages: websockets, websocket-client, alpaca-trade-api\n",
            "Successfully installed alpaca-trade-api-1.2.3 websocket-client-1.2.1 websockets-9.1\n",
            "Collecting ccxt\n",
            "  Downloading ccxt-1.55.15-py2.py3-none-any.whl (2.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0 MB 7.7 MB/s \n",
            "\u001b[?25hCollecting cryptography>=2.6.1\n",
            "  Downloading cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl (3.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2 MB 38.5 MB/s \n",
            "\u001b[?25hCollecting aiodns>=1.1.1\n",
            "  Downloading aiodns-3.0.0-py3-none-any.whl (5.0 kB)\n",
            "Requirement already satisfied: setuptools>=38.5.1 in /usr/local/lib/python3.7/dist-packages (from ccxt) (57.4.0)\n",
            "Requirement already satisfied: yarl==1.6.3 in /usr/local/lib/python3.7/dist-packages (from ccxt) (1.6.3)\n",
            "Requirement already satisfied: requests>=2.18.4 in /usr/local/lib/python3.7/dist-packages (from ccxt) (2.23.0)\n",
            "Requirement already satisfied: aiohttp<3.8,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from ccxt) (3.7.4.post0)\n",
            "Requirement already satisfied: certifi>=2018.1.18 in /usr/local/lib/python3.7/dist-packages (from ccxt) (2021.5.30)\n",
            "Requirement already satisfied: multidict>=4.0 in /usr/local/lib/python3.7/dist-packages (from yarl==1.6.3->ccxt) (5.1.0)\n",
            "Requirement already satisfied: idna>=2.0 in /usr/local/lib/python3.7/dist-packages (from yarl==1.6.3->ccxt) (2.10)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from yarl==1.6.3->ccxt) (3.7.4.3)\n",
            "Collecting pycares>=4.0.0\n",
            "  Downloading pycares-4.0.0-cp37-cp37m-manylinux2010_x86_64.whl (291 kB)\n",
            "\u001b[K     |████████████████████████████████| 291 kB 51.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: async-timeout<4.0,>=3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp<3.8,>=3.7.4->ccxt) (3.0.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp<3.8,>=3.7.4->ccxt) (21.2.0)\n",
            "Requirement already satisfied: chardet<5.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp<3.8,>=3.7.4->ccxt) (3.0.4)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.7/dist-packages (from cryptography>=2.6.1->ccxt) (1.14.6)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.12->cryptography>=2.6.1->ccxt) (2.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.18.4->ccxt) (1.24.3)\n",
            "Installing collected packages: pycares, cryptography, aiodns, ccxt\n",
            "Successfully installed aiodns-3.0.0 ccxt-1.55.15 cryptography-3.4.7 pycares-4.0.0\n",
            "Collecting jqdatasdk\n",
            "  Downloading jqdatasdk-1.8.10-py3-none-any.whl (153 kB)\n",
            "\u001b[K     |████████████████████████████████| 153 kB 8.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from jqdatasdk) (2.23.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from jqdatasdk) (1.15.0)\n",
            "Requirement already satisfied: SQLAlchemy>=1.2.8 in /usr/local/lib/python3.7/dist-packages (from jqdatasdk) (1.4.22)\n",
            "Collecting thriftpy2>=0.3.9\n",
            "  Downloading thriftpy2-0.4.14.tar.gz (361 kB)\n",
            "\u001b[K     |████████████████████████████████| 361 kB 47.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: msgpack>=0.4.7 in /usr/local/lib/python3.7/dist-packages (from jqdatasdk) (1.0.2)\n",
            "Collecting pymysql>=0.7.6\n",
            "  Downloading PyMySQL-1.0.2-py3-none-any.whl (43 kB)\n",
            "\u001b[K     |████████████████████████████████| 43 kB 2.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.9.2 in /usr/local/lib/python3.7/dist-packages (from jqdatasdk) (1.19.5)\n",
            "Requirement already satisfied: pandas>=0.16.2 in /usr/local/lib/python3.7/dist-packages (from jqdatasdk) (1.1.5)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.16.2->jqdatasdk) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.16.2->jqdatasdk) (2018.9)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from SQLAlchemy>=1.2.8->jqdatasdk) (4.6.3)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.7/dist-packages (from SQLAlchemy>=1.2.8->jqdatasdk) (1.1.1)\n",
            "Collecting ply<4.0,>=3.4\n",
            "  Downloading ply-3.11-py2.py3-none-any.whl (49 kB)\n",
            "\u001b[K     |████████████████████████████████| 49 kB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->SQLAlchemy>=1.2.8->jqdatasdk) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->SQLAlchemy>=1.2.8->jqdatasdk) (3.5.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->jqdatasdk) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->jqdatasdk) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->jqdatasdk) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->jqdatasdk) (1.24.3)\n",
            "Building wheels for collected packages: thriftpy2\n",
            "  Building wheel for thriftpy2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for thriftpy2: filename=thriftpy2-0.4.14-cp37-cp37m-linux_x86_64.whl size=940459 sha256=4743044089249d6a1d2fd93601245f329732df4b401cea2c32b5af7c91be3c26\n",
            "  Stored in directory: /root/.cache/pip/wheels/2a/f5/49/9c0d851aa64b58db72883cf9393cc824d536bdf13f5c83cff4\n",
            "Successfully built thriftpy2\n",
            "Installing collected packages: ply, thriftpy2, pymysql, jqdatasdk\n",
            "Successfully installed jqdatasdk-1.8.10 ply-3.11 pymysql-1.0.2 thriftpy2-0.4.14\n",
            "Collecting wrds\n",
            "  Downloading wrds-3.0.10-py3-none-any.whl (11 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from wrds) (1.1.5)\n",
            "Collecting psycopg2-binary\n",
            "  Downloading psycopg2_binary-2.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.4 MB 7.0 MB/s \n",
            "\u001b[?25hCollecting mock\n",
            "  Downloading mock-4.0.3-py3-none-any.whl (28 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from wrds) (1.19.5)\n",
            "Requirement already satisfied: sqlalchemy in /usr/local/lib/python3.7/dist-packages (from wrds) (1.4.22)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->wrds) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->wrds) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->wrds) (1.15.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from sqlalchemy->wrds) (4.6.3)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.7/dist-packages (from sqlalchemy->wrds) (1.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->sqlalchemy->wrds) (3.5.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->sqlalchemy->wrds) (3.7.4.3)\n",
            "Installing collected packages: psycopg2-binary, mock, wrds\n",
            "Successfully installed mock-4.0.3 psycopg2-binary-2.9.1 wrds-3.0.10\n",
            "Collecting lz4\n",
            "  Downloading lz4-3.1.3-cp37-cp37m-manylinux2010_x86_64.whl (1.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8 MB 9.6 MB/s \n",
            "\u001b[?25hInstalling collected packages: lz4\n",
            "Successfully installed lz4-3.1.3\n",
            "Requirement already satisfied: ray[tune] in /usr/local/lib/python3.7/dist-packages (1.5.2)\n",
            "Requirement already satisfied: opencensus in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (0.7.13)\n",
            "Requirement already satisfied: redis>=3.5.0 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (3.5.3)\n",
            "Requirement already satisfied: numpy>=1.16 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (2.23.0)\n",
            "Requirement already satisfied: protobuf>=3.15.3 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (3.17.3)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (3.7.4.post0)\n",
            "Requirement already satisfied: py-spy>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (0.3.8)\n",
            "Requirement already satisfied: gpustat in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (0.6.0)\n",
            "Requirement already satisfied: aioredis<2 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (1.3.1)\n",
            "Requirement already satisfied: aiohttp-cors in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (0.7.0)\n",
            "Requirement already satisfied: grpcio>=1.28.1 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (1.39.0)\n",
            "Requirement already satisfied: colorama in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (0.4.4)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (2.6.0)\n",
            "Requirement already satisfied: pydantic>=1.8 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (1.8.2)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (7.1.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (3.0.12)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (3.13)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (1.0.2)\n",
            "Requirement already satisfied: prometheus-client>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (0.11.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (0.8.9)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from ray[tune]) (1.1.5)\n",
            "Collecting tensorboardX>=1.9\n",
            "  Downloading tensorboardX-2.4-py2.py3-none-any.whl (124 kB)\n",
            "\u001b[K     |████████████████████████████████| 124 kB 10.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: hiredis in /usr/local/lib/python3.7/dist-packages (from aioredis<2->ray[tune]) (2.0.0)\n",
            "Requirement already satisfied: async-timeout in /usr/local/lib/python3.7/dist-packages (from aioredis<2->ray[tune]) (3.0.1)\n",
            "Requirement already satisfied: six>=1.5.2 in /usr/local/lib/python3.7/dist-packages (from grpcio>=1.28.1->ray[tune]) (1.15.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from pydantic>=1.8->ray[tune]) (3.7.4.3)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->ray[tune]) (5.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->ray[tune]) (1.6.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->ray[tune]) (21.2.0)\n",
            "Requirement already satisfied: chardet<5.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->ray[tune]) (3.0.4)\n",
            "Requirement already satisfied: idna>=2.0 in /usr/local/lib/python3.7/dist-packages (from yarl<2.0,>=1.0->aiohttp->ray[tune]) (2.10)\n",
            "Requirement already satisfied: blessings>=1.6 in /usr/local/lib/python3.7/dist-packages (from gpustat->ray[tune]) (1.7)\n",
            "Requirement already satisfied: nvidia-ml-py3>=7.352.0 in /usr/local/lib/python3.7/dist-packages (from gpustat->ray[tune]) (7.352.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from gpustat->ray[tune]) (5.4.8)\n",
            "Requirement already satisfied: opencensus-context==0.1.2 in /usr/local/lib/python3.7/dist-packages (from opencensus->ray[tune]) (0.1.2)\n",
            "Requirement already satisfied: google-api-core<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from opencensus->ray[tune]) (1.26.3)\n",
            "Requirement already satisfied: google-auth<2.0dev,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (1.34.0)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (1.53.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (2018.9)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (21.0)\n",
            "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (57.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (4.7.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (4.2.2)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=14.3->google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (2.4.7)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->ray[tune]) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->ray[tune]) (2021.5.30)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->ray[tune]) (2.8.2)\n",
            "Installing collected packages: tensorboardX\n",
            "Successfully installed tensorboardX-2.4\n",
            "Requirement already satisfied: tensorboardX in /usr/local/lib/python3.7/dist-packages (2.4)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from tensorboardX) (1.19.5)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX) (3.17.3)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.8.0->tensorboardX) (1.15.0)\n",
            "Collecting gputil\n",
            "  Downloading GPUtil-1.4.0.tar.gz (5.5 kB)\n",
            "Building wheels for collected packages: gputil\n",
            "  Building wheel for gputil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gputil: filename=GPUtil-1.4.0-py3-none-any.whl size=7411 sha256=13ffea6039ac238768d657af3eef3cc1c97e46572053f073453b62b840af51db\n",
            "  Stored in directory: /root/.cache/pip/wheels/6e/f8/83/534c52482d6da64622ddbf72cd93c35d2ef2881b78fd08ff0c\n",
            "Successfully built gputil\n",
            "Installing collected packages: gputil\n",
            "Successfully installed gputil-1.4.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HLZjsQH-VcOU"
      },
      "source": [
        "## Import packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N7bN2cdIDugz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "891b4112-6fa3-4fd8-e96d-aff7b41ae3c0"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "# matplotlib.use('Agg')\n",
        "import datetime\n",
        "\n",
        "from elegantrl.agent import *\n",
        "from elegantrl.run import *\n",
        "import torch \n",
        "import ray\n",
        "from finrl.apps import config\n",
        "from finrl.neo_finrl.data_processor import DataProcessor\n",
        "from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyfolio/pos.py:27: UserWarning: Module \"zipline.assets\" not found; multipliers will not be applied to position notionals.\n",
            "  'Module \"zipline.assets\" not found; multipliers will not be applied'\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I0OVWiVfXRsi"
      },
      "source": [
        "## Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "knHkXNilDug5"
      },
      "source": [
        "def train(start_date, end_date, ticker_list, data_source, time_interval, \n",
        "          technical_indicator_list, drl_lib, env, agent, if_vix = True,\n",
        "          **kwargs):\n",
        "    \n",
        "    #fetch data\n",
        "    DP = DataProcessor(data_source, **kwargs)\n",
        "    data = DP.download_data(ticker_list, start_date, end_date, time_interval)\n",
        "    data = DP.clean_data(data)\n",
        "    data = DP.add_technical_indicator(data, technical_indicator_list)\n",
        "    if if_vix:\n",
        "        data = DP.add_vix(data)\n",
        "    price_array, tech_array, turbulence_array = DP.df_to_array(data, if_vix)\n",
        "    \n",
        "    #read parameters\n",
        "    env_config = {'price_array':price_array,\n",
        "            'tech_array':tech_array,\n",
        "            'turbulence_array':turbulence_array,\n",
        "            'if_train':True}\n",
        "    env_instance = env(config=env_config)\n",
        "    \n",
        "    learning_rate = kwargs.get('learning_rate', 0.00025)\n",
        "    batch_size = kwargs.get('batch_size', 2**7)\n",
        "    gamma = kwargs.get('gamma', 0.99)\n",
        "    seed = kwargs.get('seed', 312)\n",
        "    total_timesteps = kwargs.get('total_timesteps', 1e6)\n",
        "    net_dimension = kwargs.get('net_dimension', 2**7)\n",
        "    cwd = kwargs.get('cwd','./'+str(agent))\n",
        "    \n",
        "    #train using different libraries\n",
        "    if drl_lib == 'elegantrl':\n",
        "        \n",
        "        if agent == 'ppo':\n",
        "            args = Arguments(agent=AgentPPO(), env=env_instance, if_on_policy=True)\n",
        "        else:\n",
        "            raise ValueError('Invalid agent input or the agent input is not \\\n",
        "                             supported yet.')\n",
        "        try:\n",
        "            args.cwd = cwd\n",
        "            args.learning_rate = learning_rate\n",
        "            args.batch_size = batch_size\n",
        "            args.gamma = gamma\n",
        "            args.seed = seed\n",
        "            args.break_step = total_timesteps\n",
        "            args.net_dimension = net_dimension\n",
        "        except:\n",
        "            print('Invalid parameters input! Use default value.')\n",
        "            args.learning_rate = 0.00025\n",
        "            args.batch_size = 128\n",
        "            args.gamma = 0.99\n",
        "            args.seed = 312\n",
        "            args.break_step = 1e6\n",
        "            args.net_dimension = 2**7\n",
        "            \n",
        "        train_and_evaluate(args)\n",
        "        \n",
        "    elif drl_lib == 'rllib':\n",
        "        ray.init(ignore_reinit_error=True)\n",
        "        if agent == 'ppo':\n",
        "            from ray.rllib.agents import ppo\n",
        "            from ray.rllib.agents.ppo.ppo import PPOTrainer\n",
        "            try:\n",
        "                total_episodes = kwargs.get('total_episodes')\n",
        "            except:\n",
        "                print('total training episodes is not given! Use default value 1000')\n",
        "                total_episodes = 1000\n",
        "            config = ppo.DEFAULT_CONFIG.copy()\n",
        "            config['env'] = env\n",
        "            config[\"log_level\"] = \"WARN\"\n",
        "            config['env_config'] = {'price_array':price_array,\n",
        "                                    'tech_array':tech_array,\n",
        "                                    'turbulence_array':turbulence_array,\n",
        "                                    'if_train':True}\n",
        "            trainer = PPOTrainer(env=env, config=config)\n",
        "            for i in range(total_episodes):\n",
        "                trainer.train()\n",
        "            trainer.save(cwd)\n",
        "        \n",
        "        else:\n",
        "            raise ValueError('Invalid agent input or the agent input is not \\\n",
        "                 supported yet.')\n",
        "            \n",
        "    elif drl_lib == 'stable_baselines3':\n",
        "        \n",
        "        if agent == 'ppo':\n",
        "            from stable_baselines3 import PPO\n",
        "            from stable_baselines3.common.vec_env import DummyVecEnv\n",
        "            \n",
        "            env_train = DummyVecEnv([lambda : env_instance])\n",
        "            model = PPO(\"MlpPolicy\", env_train, learning_rate=learning_rate, \n",
        "                        n_steps=2048, batch_size=batch_size, ent_coef=0.0, \n",
        "                        gamma=gamma, seed=seed)\n",
        "            model.learn(total_timesteps=total_timesteps, tb_log_name = 'ppo')\n",
        "            print('Training finished!')\n",
        "            model.save(cwd)\n",
        "            print('Trained model saved in ' + str(cwd))\n",
        "    \n",
        "    else:\n",
        "        raise ValueError('DRL library input is NOT supported. Please check.')"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FCwCJKoyXYCk"
      },
      "source": [
        "## Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lNKwj7lE04UG"
      },
      "source": [
        "def test(start_date, end_date, ticker_list, data_source, time_interval, \n",
        "         technical_indicator_list, drl_lib, env, agent, if_vix = True,\n",
        "         **kwargs):\n",
        "    #fetch data\n",
        "    DP = DataProcessor(data_source, **kwargs)\n",
        "    data = DP.download_data(ticker_list, start_date, end_date, time_interval)\n",
        "    data = DP.clean_data(data)\n",
        "    data = DP.add_technical_indicator(data, technical_indicator_list)\n",
        "\n",
        "    if if_vix:\n",
        "        data = DP.add_vix(data)\n",
        "    price_array, tech_array, turbulence_array = DP.df_to_array(data, if_vix)\n",
        "    \n",
        "    env_config = {'price_array':price_array,\n",
        "            'tech_array':tech_array,\n",
        "            'turbulence_array':turbulence_array,\n",
        "            'if_train':False}\n",
        "    env_instance = env(config=env_config)\n",
        "    \n",
        "    net_dimension = kwargs.get('net_dimension', 2**7)\n",
        "    cwd = kwargs.get('cwd','./'+str(agent))\n",
        "    print(\"price_array: \",len(price_array))\n",
        "    #test on elegantrl\n",
        "    if drl_lib == 'elegantrl':\n",
        "        \n",
        "        #select agent\n",
        "        if agent == 'ppo':\n",
        "            args = Arguments(agent=AgentPPO(), env=env_instance, if_on_policy=True)\n",
        "        else:\n",
        "            raise ValueError('Invalid agent input or the agent input is not \\\n",
        "                             supported yet.')\n",
        "        \n",
        "        #load agent\n",
        "        try:\n",
        "            state_dim = env_instance.state_dim\n",
        "            action_dim = env_instance.action_dim\n",
        "    \n",
        "            agent = args.agent\n",
        "            net_dim = net_dimension\n",
        "    \n",
        "            agent.init(net_dim, state_dim, action_dim)\n",
        "            agent.save_load_model(cwd=cwd, if_save=False)\n",
        "            act = agent.act\n",
        "            device = agent.device\n",
        "    \n",
        "        except:\n",
        "            raise ValueError('Fail to load agent!')\n",
        "        \n",
        "        #test on the testing env\n",
        "        _torch = torch\n",
        "        state = env_instance.reset()\n",
        "        episode_returns = list()  # the cumulative_return / initial_account\n",
        "        episode_total_assets = list()\n",
        "        episode_total_assets.append(env_instance.initial_total_asset)\n",
        "        with _torch.no_grad():\n",
        "            for i in range(env_instance.max_step):\n",
        "                s_tensor = _torch.as_tensor((state,), device=device)\n",
        "                a_tensor = act(s_tensor)  # action_tanh = act.forward()\n",
        "                action = a_tensor.detach().cpu().numpy()[0]  # not need detach(), because with torch.no_grad() outside\n",
        "                state, reward, done, _ = env_instance.step(action)\n",
        "\n",
        "                total_asset = env_instance.amount + (env_instance.price_ary[env_instance.day] * env_instance.stocks).sum()\n",
        "                episode_total_assets.append(total_asset)\n",
        "                episode_return = total_asset / env_instance.initial_total_asset\n",
        "                episode_returns.append(episode_return)\n",
        "                if done:\n",
        "                    break\n",
        "                ##print(i)\n",
        "        print('Test Finished!')\n",
        "        #return episode returns on testing data\n",
        "        print('episode_return', episode_return)\n",
        "        return episode_total_assets\n",
        "    \n",
        "    #test using rllib\n",
        "    elif drl_lib == 'rllib':\n",
        "        #load agent\n",
        "        from ray.rllib.agents import ppo\n",
        "        from ray.rllib.agents.ppo.ppo import PPOTrainer\n",
        "        config = ppo.DEFAULT_CONFIG.copy()\n",
        "        config['env'] = env\n",
        "        config[\"log_level\"] = \"WARN\"\n",
        "        config['env_config'] = {'price_array':price_array,\n",
        "                                'tech_array':tech_array,\n",
        "                                'turbulence_array':turbulence_array,\n",
        "                                'if_train':False}\n",
        "        \n",
        "        trainer = PPOTrainer(env=env, config=config)\n",
        "        try:\n",
        "            trainer.restore(cwd)\n",
        "            print(\"Restoring from checkpoint path\", cwd)\n",
        "        except:\n",
        "            raise ValueError('Fail to load agent!')\n",
        "        \n",
        "        #test on the testing env\n",
        "        state = env_instance.reset()\n",
        "        episode_returns = list()  # the cumulative_return / initial_account\n",
        "        episode_total_assets = list()\n",
        "        episode_total_assets.append(env_instance.initial_total_asset)\n",
        "        done = False\n",
        "        while not done:\n",
        "            action = trainer.compute_single_action(state)\n",
        "            state, reward, done, _ = env_instance.step(action)\n",
        "    \n",
        "            total_asset = env_instance.amount + (env_instance.price_ary[env_instance.day] * env_instance.stocks).sum()\n",
        "            episode_total_assets.append(total_asset)\n",
        "            episode_return = total_asset / env_instance.initial_total_asset\n",
        "            episode_returns.append(episode_return)\n",
        "        print('episode return: ' + str(episode_return))\n",
        "        print('Test Finished!')   \n",
        "        return episode_total_assets\n",
        "\n",
        "          #test using stable baselines3\n",
        "    elif drl_lib == 'stable_baselines3':\n",
        "        from stable_baselines3 import PPO\n",
        "\n",
        "        try:\n",
        "            #load agent\n",
        "            model = PPO.load(cwd)\n",
        "            print(\"Successfully load model\", cwd)\n",
        "        except:\n",
        "            raise ValueError('Fail to load agent!')\n",
        "        \n",
        "        #test on the testing env\n",
        "        state = env_instance.reset()\n",
        "        episode_returns = list()  # the cumulative_return / initial_account\n",
        "        episode_total_assets = list()\n",
        "        episode_total_assets.append(env_instance.initial_total_asset)\n",
        "        done = False\n",
        "        while not done:\n",
        "            action = model.predict(state)[0]\n",
        "            state, reward, done, _ = env_instance.step(action)\n",
        "    \n",
        "            total_asset = env_instance.amount + (env_instance.price_ary[env_instance.day] * env_instance.stocks).sum()\n",
        "            episode_total_assets.append(total_asset)\n",
        "            episode_return = total_asset / env_instance.initial_total_asset\n",
        "            episode_returns.append(episode_return)\n",
        "        \n",
        "        print('episode_return', episode_return)\n",
        "        print('Test Finished!')   \n",
        "        return episode_total_assets\n",
        "    \n",
        "    else:\n",
        "        raise ValueError('DRL library input is NOT supported yet. Please check.')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VzS1DtvaXzCB"
      },
      "source": [
        "## DRL Environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wcvNObPnDug_"
      },
      "source": [
        "from finrl.neo_finrl.env_stock_trading.env_stocktrading_np import StockTradingEnv"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "exdnEQ8eSQxZ"
      },
      "source": [
        "import numpy as np\n",
        "import os\n",
        "import gym\n",
        "from numpy import random as rd\n",
        "\n",
        "class StockTradingEnv(gym.Env):\n",
        "\n",
        "    def __init__(self, config, initial_account=1e6,\n",
        "                 gamma=0.99, turbulence_thresh=99, min_stock_rate=0.1,\n",
        "                 max_stock=1e2, initial_capital=1e6, buy_cost_pct=1e-3, \n",
        "                 sell_cost_pct=1e-3,reward_scaling=2 ** -11,  initial_stocks=None,\n",
        "                 ):\n",
        "        price_ary = config['price_array']\n",
        "        tech_ary = config['tech_array']\n",
        "        turbulence_ary = config['turbulence_array']\n",
        "        if_train = config['if_train']\n",
        "        n = price_ary.shape[0]\n",
        "        self.price_ary =  price_ary.astype(np.float32)\n",
        "        self.tech_ary = tech_ary.astype(np.float32)\n",
        "        self.turbulence_ary = turbulence_ary\n",
        "        \n",
        "        self.tech_ary = self.tech_ary * 2 ** -7\n",
        "        self.turbulence_bool = (turbulence_ary > turbulence_thresh).astype(np.float32)\n",
        "        self.turbulence_ary = (self.sigmoid_sign(turbulence_ary, turbulence_thresh) * 2 ** -5).astype(np.float32)\n",
        "\n",
        "        stock_dim = self.price_ary.shape[1]\n",
        "        self.gamma = gamma\n",
        "        self.max_stock = max_stock\n",
        "        self.min_stock_rate = min_stock_rate\n",
        "        self.buy_cost_pct = buy_cost_pct\n",
        "        self.sell_cost_pct = sell_cost_pct\n",
        "        self.reward_scaling = reward_scaling\n",
        "        self.initial_capital = initial_capital\n",
        "        self.initial_stocks = np.zeros(stock_dim, dtype=np.float32) if initial_stocks is None else initial_stocks\n",
        "\n",
        "        # reset()\n",
        "        self.day = None\n",
        "        self.amount = None\n",
        "        self.stocks = None\n",
        "        self.total_asset = None\n",
        "        self.gamma_reward = None\n",
        "        self.initial_total_asset = None\n",
        "\n",
        "        # environment information\n",
        "        self.env_name = 'StockEnv'\n",
        "        # self.state_dim = 1 + 2 + 2 * stock_dim + self.tech_ary.shape[1]\n",
        "        # # amount + (turbulence, turbulence_bool) + (price, stock) * stock_dim + tech_dim\n",
        "        self.state_dim = 1 + 2 + 3 * stock_dim + self.tech_ary.shape[1]\n",
        "        # amount + (turbulence, turbulence_bool) + (price, stock) * stock_dim + tech_dim\n",
        "        self.stocks_cd = None\n",
        "        self.action_dim = stock_dim\n",
        "        self.max_step = self.price_ary.shape[0] - 1\n",
        "        self.if_train = if_train\n",
        "        self.if_discrete = False\n",
        "        self.target_return = 5.0\n",
        "        self.episode_return = 0.0\n",
        "        \n",
        "        self.observation_space = gym.spaces.Box(low=-3000, high=3000, shape=(self.state_dim,), dtype=np.float32)\n",
        "        self.action_space = gym.spaces.Box(low=-1, high=1, shape=(self.action_dim,), dtype=np.float32)\n",
        "        \n",
        "    def reset(self):\n",
        "        self.day = 0\n",
        "        price = self.price_ary[self.day]\n",
        "        \n",
        "        if self.if_train:\n",
        "            self.stocks = (self.initial_stocks + rd.randint(0, 64, size=self.initial_stocks.shape)).astype(np.float32)\n",
        "            self.stocks_cd = np.zeros_like(self.stocks)\n",
        "            self.amount = self.initial_capital * rd.uniform(0.95, 1.05) - (self.stocks * price).sum()\n",
        "        else:\n",
        "            self.stocks = self.initial_stocks.astype(np.float32)\n",
        "            self.stocks_cd = np.zeros_like(self.stocks)\n",
        "            self.amount = self.initial_capital\n",
        "\n",
        "        self.total_asset = self.amount + (self.stocks * price).sum()\n",
        "        self.initial_total_asset = self.total_asset\n",
        "        self.gamma_reward = 0.0\n",
        "        return self.get_state(price)  # state\n",
        "\n",
        "    def step(self, actions):\n",
        "        actions = (actions * self.max_stock).astype(int)\n",
        "\n",
        "        self.day += 1\n",
        "        price = self.price_ary[self.day]\n",
        "        self.stocks_cd += 1\n",
        "\n",
        "        if self.turbulence_bool[self.day] == 0:\n",
        "            min_action = int(self.max_stock * self.min_stock_rate)  # stock_cd\n",
        "            for index in np.where(actions < -min_action)[0]:  # sell_index:\n",
        "                if price[index] > 0:  # Sell only if current asset is > 0\n",
        "                    sell_num_shares = min(self.stocks[index], -actions[index])\n",
        "                    self.stocks[index] -= sell_num_shares\n",
        "                    self.amount += price[index] * sell_num_shares * (1 - self.sell_cost_pct)\n",
        "                    self.stocks_cd[index] = 0\n",
        "            for index in np.where(actions > min_action)[0]:  # buy_index:\n",
        "                if price[index] > 0:  # Buy only if the price is > 0 (no missing data in this particular date)\n",
        "                    buy_num_shares = min(self.amount // price[index], actions[index])\n",
        "                    self.stocks[index] += buy_num_shares\n",
        "                    self.amount -= price[index] * buy_num_shares * (1 + self.buy_cost_pct)\n",
        "                    self.stocks_cd[index] = 0\n",
        "\n",
        "        else:  # sell all when turbulence\n",
        "            self.amount += (self.stocks * price).sum() * (1 - self.sell_cost_pct)\n",
        "            self.stocks[:] = 0\n",
        "            self.stocks_cd[:] = 0\n",
        "\n",
        "        state = self.get_state(price)\n",
        "        total_asset = self.amount + (self.stocks * price).sum()\n",
        "        reward = (total_asset - self.total_asset) * self.reward_scaling\n",
        "        self.total_asset = total_asset\n",
        "\n",
        "        self.gamma_reward = self.gamma_reward * self.gamma + reward\n",
        "        done = self.day == self.max_step\n",
        "        if done:\n",
        "            reward = self.gamma_reward\n",
        "            self.episode_return = total_asset / self.initial_total_asset\n",
        "\n",
        "        return state, reward, done, dict()\n",
        "\n",
        "    def get_state(self, price):\n",
        "        amount = np.array(max(self.amount, 1e4) * (2 ** -12), dtype=np.float32)\n",
        "        scale = np.array(2 ** -6, dtype=np.float32)\n",
        "        return np.hstack((amount,\n",
        "                          self.turbulence_ary[self.day],\n",
        "                          self.turbulence_bool[self.day],\n",
        "                          price * scale,\n",
        "                          self.stocks * scale,\n",
        "                          self.stocks_cd,\n",
        "                          self.tech_ary[self.day],\n",
        "                          ))  # state.astype(np.float32)\n",
        "    \n",
        "    @staticmethod\n",
        "    def sigmoid_sign(ary, thresh):\n",
        "        def sigmoid(x):\n",
        "            return 1 / (1 + np.exp(-x * np.e)) - 0.5\n",
        "\n",
        "        return sigmoid(ary / thresh) * thresh\n"
      ],
      "execution_count": 85,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LcbH9Ah9X7vw"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lM3lVwmQX8GA"
      },
      "source": [
        "## Set parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l5dYmiPpDug_"
      },
      "source": [
        "env = StockTradingEnv"
      ],
      "execution_count": 87,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXgk2XftXwoR"
      },
      "source": [
        "TRAIN_START_DATE = '2014-01-01'\n",
        "TRAIN_END_DATE = '2020-07-30'\n",
        "\n",
        "TEST_START_DATE = '2020-08-01'\n",
        "TEST_END_DATE = '2021-08-05'\n",
        "TECHNICAL_INDICATORS_LIST = ['macd',\n",
        " 'boll_ub',\n",
        " 'boll_lb',\n",
        " 'rsi_30',\n",
        " 'dx_30',\n",
        " 'close_30_sma',\n",
        " 'close_60_sma']"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1TC8cMZaYAG1"
      },
      "source": [
        "## eRL"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tO1sXcIJZdtE"
      },
      "source": [
        "### Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JodM4Qn3Dug_",
        "outputId": "175afd5f-859e-4409-904d-97b264ffcb56"
      },
      "source": [
        "#demo for elegantrl\n",
        "train(start_date = TRAIN_START_DATE, \n",
        "      end_date = TRAIN_END_DATE,\n",
        "      ticker_list = config.DOW_30_TICKER, \n",
        "      data_source = 'yahoofinance',\n",
        "      time_interval= '1D', \n",
        "      technical_indicator_list= TECHNICAL_INDICATORS_LIST,\n",
        "      drl_lib='elegantrl', \n",
        "      env=env, \n",
        "      agent='ppo', \n",
        "      cwd='./test_ppo'\n",
        "      ,total_timesteps=3e4)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (49650, 9)\n",
            "Clean data for AAPL\n",
            "Data clean for AAPL is finished.\n",
            "Clean data for AXP\n",
            "Data clean for AXP is finished.\n",
            "Clean data for BA\n",
            "Data clean for BA is finished.\n",
            "Clean data for CAT\n",
            "Data clean for CAT is finished.\n",
            "Clean data for CSCO\n",
            "Data clean for CSCO is finished.\n",
            "Clean data for CVX\n",
            "Data clean for CVX is finished.\n",
            "Clean data for DD\n",
            "Data clean for DD is finished.\n",
            "Clean data for DIS\n",
            "Data clean for DIS is finished.\n",
            "Clean data for GS\n",
            "Data clean for GS is finished.\n",
            "Clean data for HD\n",
            "Data clean for HD is finished.\n",
            "Clean data for IBM\n",
            "Data clean for IBM is finished.\n",
            "Clean data for INTC\n",
            "Data clean for INTC is finished.\n",
            "Clean data for JNJ\n",
            "Data clean for JNJ is finished.\n",
            "Clean data for JPM\n",
            "Data clean for JPM is finished.\n",
            "Clean data for KO\n",
            "Data clean for KO is finished.\n",
            "Clean data for MCD\n",
            "Data clean for MCD is finished.\n",
            "Clean data for MMM\n",
            "Data clean for MMM is finished.\n",
            "Clean data for MRK\n",
            "Data clean for MRK is finished.\n",
            "Clean data for MSFT\n",
            "Data clean for MSFT is finished.\n",
            "Clean data for NKE\n",
            "Data clean for NKE is finished.\n",
            "Clean data for PFE\n",
            "Data clean for PFE is finished.\n",
            "Clean data for PG\n",
            "Data clean for PG is finished.\n",
            "Clean data for RTX\n",
            "Data clean for RTX is finished.\n",
            "Clean data for TRV\n",
            "Data clean for TRV is finished.\n",
            "Clean data for UNH\n",
            "Data clean for UNH is finished.\n",
            "Clean data for V\n",
            "Data clean for V is finished.\n",
            "Clean data for VZ\n",
            "Data clean for VZ is finished.\n",
            "Clean data for WBA\n",
            "Data clean for WBA is finished.\n",
            "Clean data for WMT\n",
            "Data clean for WMT is finished.\n",
            "Clean data for XOM\n",
            "Data clean for XOM is finished.\n",
            "Data clean all finished!\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (1655, 9)\n",
            "Clean data for ^VIX\n",
            "Data clean for ^VIX is finished.\n",
            "Data clean all finished!\n",
            "['AAPL' 'AXP' 'BA' 'CAT' 'CSCO' 'CVX' 'DD' 'DIS' 'GS' 'HD' 'IBM' 'INTC'\n",
            " 'JNJ' 'JPM' 'KO' 'MCD' 'MMM' 'MRK' 'MSFT' 'NKE' 'PFE' 'PG' 'RTX' 'TRV'\n",
            " 'UNH' 'V' 'VZ' 'WBA' 'WMT' 'XOM']\n",
            "Successfully transformed into array\n",
            "| GPU id: 0, cwd: ./test_ppo\n",
            "| Remove history\n",
            "ID      Step      MaxR |    avgR      stdR       objA      objC |  avgS  stdS\n",
            "0   4.96e+03      2.78 |\n",
            "0   4.96e+03      2.78 |    2.78      0.08      -0.15      9.37 |  1655     0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9eSQ64_nZhJ1"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZOdNzZnoYhH3",
        "outputId": "a336f0e0-4a5d-44b6-b3e4-498370afaa4e"
      },
      "source": [
        "account_value_erl=test(start_date = TEST_START_DATE, \n",
        "                        end_date = TEST_END_DATE,\n",
        "                        ticker_list = config.DOW_30_TICKER, \n",
        "                        data_source = 'yahoofinance',\n",
        "                        time_interval= '1D', \n",
        "                        technical_indicator_list= TECHNICAL_INDICATORS_LIST,\n",
        "                        drl_lib='elegantrl', \n",
        "                        env=env, \n",
        "                        agent='ppo', \n",
        "                        cwd='./test_ppo', \n",
        "                        net_dimension = 2 ** 9)"
      ],
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (7620, 9)\n",
            "Clean data for AAPL\n",
            "Data clean for AAPL is finished.\n",
            "Clean data for AXP\n",
            "Data clean for AXP is finished.\n",
            "Clean data for BA\n",
            "Data clean for BA is finished.\n",
            "Clean data for CAT\n",
            "Data clean for CAT is finished.\n",
            "Clean data for CSCO\n",
            "Data clean for CSCO is finished.\n",
            "Clean data for CVX\n",
            "Data clean for CVX is finished.\n",
            "Clean data for DD\n",
            "Data clean for DD is finished.\n",
            "Clean data for DIS\n",
            "Data clean for DIS is finished.\n",
            "Clean data for GS\n",
            "Data clean for GS is finished.\n",
            "Clean data for HD\n",
            "Data clean for HD is finished.\n",
            "Clean data for IBM\n",
            "Data clean for IBM is finished.\n",
            "Clean data for INTC\n",
            "Data clean for INTC is finished.\n",
            "Clean data for JNJ\n",
            "Data clean for JNJ is finished.\n",
            "Clean data for JPM\n",
            "Data clean for JPM is finished.\n",
            "Clean data for KO\n",
            "Data clean for KO is finished.\n",
            "Clean data for MCD\n",
            "Data clean for MCD is finished.\n",
            "Clean data for MMM\n",
            "Data clean for MMM is finished.\n",
            "Clean data for MRK\n",
            "Data clean for MRK is finished.\n",
            "Clean data for MSFT\n",
            "Data clean for MSFT is finished.\n",
            "Clean data for NKE\n",
            "Data clean for NKE is finished.\n",
            "Clean data for PFE\n",
            "Data clean for PFE is finished.\n",
            "Clean data for PG\n",
            "Data clean for PG is finished.\n",
            "Clean data for RTX\n",
            "Data clean for RTX is finished.\n",
            "Clean data for TRV\n",
            "Data clean for TRV is finished.\n",
            "Clean data for UNH\n",
            "Data clean for UNH is finished.\n",
            "Clean data for V\n",
            "Data clean for V is finished.\n",
            "Clean data for VZ\n",
            "Data clean for VZ is finished.\n",
            "Clean data for WBA\n",
            "Data clean for WBA is finished.\n",
            "Clean data for WMT\n",
            "Data clean for WMT is finished.\n",
            "Clean data for XOM\n",
            "Data clean for XOM is finished.\n",
            "Data clean all finished!\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (254, 9)\n",
            "Clean data for ^VIX\n",
            "Data clean for ^VIX is finished.\n",
            "Data clean all finished!\n",
            "['AAPL' 'AXP' 'BA' 'CAT' 'CSCO' 'CVX' 'DD' 'DIS' 'GS' 'HD' 'IBM' 'INTC'\n",
            " 'JNJ' 'JPM' 'KO' 'MCD' 'MMM' 'MRK' 'MSFT' 'NKE' 'PFE' 'PG' 'RTX' 'TRV'\n",
            " 'UNH' 'V' 'VZ' 'WBA' 'WMT' 'XOM']\n",
            "Successfully transformed into array\n",
            "price_array:  255\n",
            "Loaded act: ./test_ppo\n",
            "Test Finished!\n",
            "episode_return 1.2745876945037917\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z-u8JqQbshMj",
        "outputId": "84a47d2f-8aa9-443e-a46d-c0520799e254"
      },
      "source": [
        "len(account_value_erl)"
      ],
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "255"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8F8Xqq6rjgly"
      },
      "source": [
        "### Plot"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "Koa7NilB7-Ge",
        "outputId": "f819d797-e717-4af8-e92b-2efa123fee28"
      },
      "source": [
        "TEST_END_DATE"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'2021-08-05'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uzlZfeHucDIh",
        "outputId": "d482b3c5-d4dc-4c51-ba90-b60428f690b7"
      },
      "source": [
        "baseline_df =  DataProcessor('yahoofinance').download_data(ticker_list = [\"^DJI\"], \n",
        "                                                            start_date = TEST_START_DATE, \n",
        "                                                            end_date = TEST_END_DATE, \n",
        "                                                            time_interval = \"1D\")\n",
        "stats = backtest_stats(baseline_df, value_col_name = 'close')\n"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (254, 9)\n",
            "Annual return          0.302105\n",
            "Cumulative returns     0.304836\n",
            "Annual volatility      0.143039\n",
            "Sharpe ratio           1.925145\n",
            "Calmar ratio           3.382743\n",
            "Stability              0.935738\n",
            "Max drawdown          -0.089308\n",
            "Omega ratio            1.381054\n",
            "Sortino ratio          2.847623\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.074665\n",
            "Daily value at risk   -0.016928\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h9wmxqC5sqZN"
      },
      "source": [
        "account_value_erl = pd.DataFrame({'date':baseline_df.date,'account_value':account_value_erl[0:len(account_value_erl)-1]})"
      ],
      "execution_count": 95,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "EoHB3xzY6GrR",
        "outputId": "26e045fc-2ccc-40f3-c689-a9d10c7620b4"
      },
      "source": [
        "account_value_erl.tail()"
      ],
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>account_value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>249</th>\n",
              "      <td>2021-07-29</td>\n",
              "      <td>1.263132e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>250</th>\n",
              "      <td>2021-07-30</td>\n",
              "      <td>1.264554e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>251</th>\n",
              "      <td>2021-08-02</td>\n",
              "      <td>1.264451e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>252</th>\n",
              "      <td>2021-08-03</td>\n",
              "      <td>1.280739e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>253</th>\n",
              "      <td>2021-08-04</td>\n",
              "      <td>1.274588e+06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           date  account_value\n",
              "249  2021-07-29   1.263132e+06\n",
              "250  2021-07-30   1.264554e+06\n",
              "251  2021-08-02   1.264451e+06\n",
              "252  2021-08-03   1.280739e+06\n",
              "253  2021-08-04   1.274588e+06"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3lZjaI5fbLQC",
        "outputId": "296f7557-8d20-4f78-fb02-57ad9123abc7"
      },
      "source": [
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all = backtest_stats(account_value=account_value_erl)\n",
        "perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "perf_stats_all.to_csv(\"./\"+\"/perf_stats_all_\"+now+'.csv')"
      ],
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.272155\n",
            "Cumulative returns     0.274588\n",
            "Annual volatility      0.139828\n",
            "Sharpe ratio           1.798816\n",
            "Calmar ratio           3.306316\n",
            "Stability              0.929235\n",
            "Max drawdown          -0.082314\n",
            "Omega ratio            1.361410\n",
            "Sortino ratio          2.694458\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.017422\n",
            "Daily value at risk   -0.016619\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9qJUg0KTBIGw"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wZ1A4lB7jZ2r",
        "outputId": "aa348f6b-0050-4305-f10f-ce66959b88d4"
      },
      "source": [
        "print(\"==============Compare to DJIA===========\")\n",
        "%matplotlib inline\n",
        "# S&P 500: ^GSPC\n",
        "# Dow Jones Index: ^DJI\n",
        "# NASDAQ 100: ^NDX\n",
        "backtest_plot(account_value_erl, \n",
        "             baseline_ticker = '^DJI', \n",
        "             baseline_start = account_value_erl.loc[0,'date'],\n",
        "             baseline_end = account_value_erl.loc[len(account_value_erl)-1,'date'])"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "==============Compare to DJIA===========\n",
            "\r[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (253, 8)\n",
            "<IPython.core.display.HTML object>\n",
            "<IPython.core.display.HTML object>\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyfolio/tears.py:907: UserWarning:\n",
            "\n",
            "Passed returns do not overlap with anyinteresting times.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "<Figure size 1008x5184 with 13 Axes>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VzTdYfbNCKiE"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aq6QHtD4YuE5"
      },
      "source": [
        "## RLlib"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OVMEflpTZjc0"
      },
      "source": [
        "### Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MOuD9mdLDuhA",
        "outputId": "74ea9928-20ff-4ec7-91db-3ada56d781f7"
      },
      "source": [
        "#demo for rllib\n",
        "train(start_date = TEST_START_DATE, \n",
        "      end_date = TEST_END_DATE,\n",
        "      ticker_list = config.DOW_30_TICKER, \n",
        "      data_source = 'yahoofinance',\n",
        "      time_interval= '1D', \n",
        "      technical_indicator_list= TECHNICAL_INDICATORS_LIST,\n",
        "      drl_lib='rllib', \n",
        "      env=env, \n",
        "      agent='ppo', \n",
        "      cwd='./test_ppo'\n",
        "      ,total_episodes=100)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (7560, 9)\n",
            "Clean data for AAPL\n",
            "Data clean for AAPL is finished.\n",
            "Clean data for AXP\n",
            "Data clean for AXP is finished.\n",
            "Clean data for BA\n",
            "Data clean for BA is finished.\n",
            "Clean data for CAT\n",
            "Data clean for CAT is finished.\n",
            "Clean data for CSCO\n",
            "Data clean for CSCO is finished.\n",
            "Clean data for CVX\n",
            "Data clean for CVX is finished.\n",
            "Clean data for DD\n",
            "Data clean for DD is finished.\n",
            "Clean data for DIS\n",
            "Data clean for DIS is finished.\n",
            "Clean data for GS\n",
            "Data clean for GS is finished.\n",
            "Clean data for HD\n",
            "Data clean for HD is finished.\n",
            "Clean data for IBM\n",
            "Data clean for IBM is finished.\n",
            "Clean data for INTC\n",
            "Data clean for INTC is finished.\n",
            "Clean data for JNJ\n",
            "Data clean for JNJ is finished.\n",
            "Clean data for JPM\n",
            "Data clean for JPM is finished.\n",
            "Clean data for KO\n",
            "Data clean for KO is finished.\n",
            "Clean data for MCD\n",
            "Data clean for MCD is finished.\n",
            "Clean data for MMM\n",
            "Data clean for MMM is finished.\n",
            "Clean data for MRK\n",
            "Data clean for MRK is finished.\n",
            "Clean data for MSFT\n",
            "Data clean for MSFT is finished.\n",
            "Clean data for NKE\n",
            "Data clean for NKE is finished.\n",
            "Clean data for PFE\n",
            "Data clean for PFE is finished.\n",
            "Clean data for PG\n",
            "Data clean for PG is finished.\n",
            "Clean data for RTX\n",
            "Data clean for RTX is finished.\n",
            "Clean data for TRV\n",
            "Data clean for TRV is finished.\n",
            "Clean data for UNH\n",
            "Data clean for UNH is finished.\n",
            "Clean data for V\n",
            "Data clean for V is finished.\n",
            "Clean data for VZ\n",
            "Data clean for VZ is finished.\n",
            "Clean data for WBA\n",
            "Data clean for WBA is finished.\n",
            "Clean data for WMT\n",
            "Data clean for WMT is finished.\n",
            "Clean data for XOM\n",
            "Data clean for XOM is finished.\n",
            "Data clean all finished!\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (252, 9)\n",
            "Clean data for ^VIX\n",
            "Data clean for ^VIX is finished.\n",
            "Data clean all finished!\n",
            "['AAPL' 'AXP' 'BA' 'CAT' 'CSCO' 'CVX' 'DD' 'DIS' 'GS' 'HD' 'IBM' 'INTC'\n",
            " 'JNJ' 'JPM' 'KO' 'MCD' 'MMM' 'MRK' 'MSFT' 'NKE' 'PFE' 'PG' 'RTX' 'TRV'\n",
            " 'UNH' 'V' 'VZ' 'WBA' 'WMT' 'XOM']\n",
            "Successfully transformed into array\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-08-18 13:47:10,349\tINFO services.py:1247 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n",
            "2021-08-18 13:47:21,858\tINFO trainer.py:706 -- Tip: set framework=tfe or the --eager flag to enable TensorFlow eager execution\n",
            "2021-08-18 13:47:21,860\tINFO trainer.py:720 -- Current log_level is WARN. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.\n",
            "2021-08-18 13:47:38,993\tINFO trainable.py:109 -- Trainable.setup took 17.138 seconds. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lpMmJAkfZlMx"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XtDjEqTcYwXQ",
        "outputId": "40c21f88-3ad9-4936-d857-ef9a602aafb1"
      },
      "source": [
        "account_value_rllib = test(start_date = TEST_START_DATE, \n",
        "     end_date = TEST_END_DATE,\n",
        "     ticker_list = config.DOW_30_TICKER, \n",
        "     data_source = 'yahoofinance',\n",
        "     time_interval= '1D', \n",
        "     technical_indicator_list= TECHNICAL_INDICATORS_LIST,\n",
        "     drl_lib='rllib', \n",
        "     env=env, \n",
        "     agent='ppo', \n",
        "     cwd='./test_ppo/checkpoint_000100/checkpoint-100')"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (7620, 9)\n",
            "Clean data for AAPL\n",
            "Data clean for AAPL is finished.\n",
            "Clean data for AXP\n",
            "Data clean for AXP is finished.\n",
            "Clean data for BA\n",
            "Data clean for BA is finished.\n",
            "Clean data for CAT\n",
            "Data clean for CAT is finished.\n",
            "Clean data for CSCO\n",
            "Data clean for CSCO is finished.\n",
            "Clean data for CVX\n",
            "Data clean for CVX is finished.\n",
            "Clean data for DD\n",
            "Data clean for DD is finished.\n",
            "Clean data for DIS\n",
            "Data clean for DIS is finished.\n",
            "Clean data for GS\n",
            "Data clean for GS is finished.\n",
            "Clean data for HD\n",
            "Data clean for HD is finished.\n",
            "Clean data for IBM\n",
            "Data clean for IBM is finished.\n",
            "Clean data for INTC\n",
            "Data clean for INTC is finished.\n",
            "Clean data for JNJ\n",
            "Data clean for JNJ is finished.\n",
            "Clean data for JPM\n",
            "Data clean for JPM is finished.\n",
            "Clean data for KO\n",
            "Data clean for KO is finished.\n",
            "Clean data for MCD\n",
            "Data clean for MCD is finished.\n",
            "Clean data for MMM\n",
            "Data clean for MMM is finished.\n",
            "Clean data for MRK\n",
            "Data clean for MRK is finished.\n",
            "Clean data for MSFT\n",
            "Data clean for MSFT is finished.\n",
            "Clean data for NKE\n",
            "Data clean for NKE is finished.\n",
            "Clean data for PFE\n",
            "Data clean for PFE is finished.\n",
            "Clean data for PG\n",
            "Data clean for PG is finished.\n",
            "Clean data for RTX\n",
            "Data clean for RTX is finished.\n",
            "Clean data for TRV\n",
            "Data clean for TRV is finished.\n",
            "Clean data for UNH\n",
            "Data clean for UNH is finished.\n",
            "Clean data for V\n",
            "Data clean for V is finished.\n",
            "Clean data for VZ\n",
            "Data clean for VZ is finished.\n",
            "Clean data for WBA\n",
            "Data clean for WBA is finished.\n",
            "Clean data for WMT\n",
            "Data clean for WMT is finished.\n",
            "Clean data for XOM\n",
            "Data clean for XOM is finished.\n",
            "Data clean all finished!\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (254, 9)\n",
            "Clean data for ^VIX\n",
            "Data clean for ^VIX is finished.\n",
            "Data clean all finished!\n",
            "['AAPL' 'AXP' 'BA' 'CAT' 'CSCO' 'CVX' 'DD' 'DIS' 'GS' 'HD' 'IBM' 'INTC'\n",
            " 'JNJ' 'JPM' 'KO' 'MCD' 'MMM' 'MRK' 'MSFT' 'NKE' 'PFE' 'PG' 'RTX' 'TRV'\n",
            " 'UNH' 'V' 'VZ' 'WBA' 'WMT' 'XOM']\n",
            "Successfully transformed into array\n",
            "price_array:  255\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-08-18 14:24:22,130\tINFO trainable.py:109 -- Trainable.setup took 19.542 seconds. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n",
            "2021-08-18 14:24:22,331\tINFO trainable.py:383 -- Restored on 172.28.0.2 from checkpoint: ./test_ppo/checkpoint_000100/checkpoint-100\n",
            "2021-08-18 14:24:22,339\tINFO trainable.py:390 -- Current state after restoring: {'_iteration': 100, '_timesteps_total': None, '_time_total': 2031.5027058124542, '_episodes_total': 1586}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Restoring from checkpoint path ./test_ppo/checkpoint_000100/checkpoint-100\n",
            "episode return: 1.7326374529744506\n",
            "Test Finished!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H6yq9e776g3z",
        "outputId": "2b89f981-08c6-4bfa-ca41-e1831f21ba5f"
      },
      "source": [
        "len(account_value_rllib)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "255"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B-RC3cjI2X1a"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k6ajBhvMBdsG"
      },
      "source": [
        "### Plot"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gEmVTLNBh6-F",
        "outputId": "535ef8b9-0d22-4a82-afc9-a4e9e7d797a5"
      },
      "source": [
        "baseline_df =  DataProcessor('yahoofinance').download_data(ticker_list = [\"^DJI\"], \n",
        "                                                            start_date = TEST_START_DATE, \n",
        "                                                            end_date = TEST_END_DATE, \n",
        "                                                            time_interval = \"1D\")\n",
        "stats = backtest_stats(baseline_df, value_col_name = 'close')\n"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (254, 9)\n",
            "Annual return          0.302105\n",
            "Cumulative returns     0.304836\n",
            "Annual volatility      0.143039\n",
            "Sharpe ratio           1.925145\n",
            "Calmar ratio           3.382743\n",
            "Stability              0.935738\n",
            "Max drawdown          -0.089308\n",
            "Omega ratio            1.381054\n",
            "Sortino ratio          2.847623\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.074665\n",
            "Daily value at risk   -0.016928\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qhtuzazbponh",
        "outputId": "f6663c2e-7dd4-4bc8-b962-d8d35609e688"
      },
      "source": [
        "len(baseline_df.date)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "254"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-v5aoZ10Bgky"
      },
      "source": [
        "account_value_rllib = pd.DataFrame({'date':baseline_df.date,'account_value':account_value_rllib[0:len(account_value_rllib)-1]})"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MTtGKUwwiAPf",
        "outputId": "15f7a1a3-8f58-4200-d815-b58089b81ca2"
      },
      "source": [
        "perf_stats_all = backtest_stats(account_value=account_value_rllib)\n",
        "perf_stats_all = pd.DataFrame(perf_stats_all)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Annual return          0.725243\n",
            "Cumulative returns     0.732726\n",
            "Annual volatility      0.210236\n",
            "Sharpe ratio           2.711412\n",
            "Calmar ratio           7.819859\n",
            "Stability              0.955615\n",
            "Max drawdown          -0.092744\n",
            "Omega ratio            1.572524\n",
            "Sortino ratio          4.526614\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.091560\n",
            "Daily value at risk   -0.024225\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pSAgvG7CBnIJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ebae53f0-e2a9-4576-ec64-76d6ae20a266"
      },
      "source": [
        "print(\"==============Compare to DJIA===========\")\n",
        "%matplotlib inline\n",
        "# S&P 500: ^GSPC\n",
        "# Dow Jones Index: ^DJI\n",
        "# NASDAQ 100: ^NDX\n",
        "backtest_plot(account_value_rllib, \n",
        "             baseline_ticker = '^DJI', \n",
        "             baseline_start = account_value_rllib.loc[0,'date'],\n",
        "             baseline_end = account_value_rllib.loc[len(account_value_rllib)-1,'date'])"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "==============Compare to DJIA===========\n",
            "\r[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (253, 8)\n",
            "<IPython.core.display.HTML object>\n",
            "<IPython.core.display.HTML object>\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyfolio/tears.py:907: UserWarning: Passed returns do not overlap with anyinteresting times.\n",
            "  'interesting times.', UserWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "<Figure size 1008x5184 with 13 Axes>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hN_2gi3SY6U1"
      },
      "source": [
        "## Stable-baselines3"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aRBk-NDdZmx0"
      },
      "source": [
        "### Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2A7msCg0DuhA",
        "outputId": "a0a769bb-44f8-4761-ba04-bf54db8d0612"
      },
      "source": [
        "#demo for stable-baselines3\n",
        "train(start_date = TRAIN_START_DATE, \n",
        "      end_date = TRAIN_END_DATE,\n",
        "      ticker_list = config.DOW_30_TICKER, \n",
        "      data_source = 'yahoofinance',\n",
        "      time_interval= '1D', \n",
        "      technical_indicator_list= TECHNICAL_INDICATORS_LIST,\n",
        "      drl_lib='stable_baselines3', \n",
        "      env=env, \n",
        "      agent='ppo', \n",
        "      cwd='./test_ppo',\n",
        "      total_timesteps=100000)\n"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (49650, 9)\n",
            "Clean data for AAPL\n",
            "Data clean for AAPL is finished.\n",
            "Clean data for AXP\n",
            "Data clean for AXP is finished.\n",
            "Clean data for BA\n",
            "Data clean for BA is finished.\n",
            "Clean data for CAT\n",
            "Data clean for CAT is finished.\n",
            "Clean data for CSCO\n",
            "Data clean for CSCO is finished.\n",
            "Clean data for CVX\n",
            "Data clean for CVX is finished.\n",
            "Clean data for DD\n",
            "Data clean for DD is finished.\n",
            "Clean data for DIS\n",
            "Data clean for DIS is finished.\n",
            "Clean data for GS\n",
            "Data clean for GS is finished.\n",
            "Clean data for HD\n",
            "Data clean for HD is finished.\n",
            "Clean data for IBM\n",
            "Data clean for IBM is finished.\n",
            "Clean data for INTC\n",
            "Data clean for INTC is finished.\n",
            "Clean data for JNJ\n",
            "Data clean for JNJ is finished.\n",
            "Clean data for JPM\n",
            "Data clean for JPM is finished.\n",
            "Clean data for KO\n",
            "Data clean for KO is finished.\n",
            "Clean data for MCD\n",
            "Data clean for MCD is finished.\n",
            "Clean data for MMM\n",
            "Data clean for MMM is finished.\n",
            "Clean data for MRK\n",
            "Data clean for MRK is finished.\n",
            "Clean data for MSFT\n",
            "Data clean for MSFT is finished.\n",
            "Clean data for NKE\n",
            "Data clean for NKE is finished.\n",
            "Clean data for PFE\n",
            "Data clean for PFE is finished.\n",
            "Clean data for PG\n",
            "Data clean for PG is finished.\n",
            "Clean data for RTX\n",
            "Data clean for RTX is finished.\n",
            "Clean data for TRV\n",
            "Data clean for TRV is finished.\n",
            "Clean data for UNH\n",
            "Data clean for UNH is finished.\n",
            "Clean data for V\n",
            "Data clean for V is finished.\n",
            "Clean data for VZ\n",
            "Data clean for VZ is finished.\n",
            "Clean data for WBA\n",
            "Data clean for WBA is finished.\n",
            "Clean data for WMT\n",
            "Data clean for WMT is finished.\n",
            "Clean data for XOM\n",
            "Data clean for XOM is finished.\n",
            "Data clean all finished!\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (1655, 9)\n",
            "Clean data for ^VIX\n",
            "Data clean for ^VIX is finished.\n",
            "Data clean all finished!\n",
            "['AAPL' 'AXP' 'BA' 'CAT' 'CSCO' 'CVX' 'DD' 'DIS' 'GS' 'HD' 'IBM' 'INTC'\n",
            " 'JNJ' 'JPM' 'KO' 'MCD' 'MMM' 'MRK' 'MSFT' 'NKE' 'PFE' 'PG' 'RTX' 'TRV'\n",
            " 'UNH' 'V' 'VZ' 'WBA' 'WMT' 'XOM']\n",
            "Successfully transformed into array\n",
            "Training finished!\n",
            "Trained model saved in ./test_ppo\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TsAByoEOZn6X"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fKEYTIcLQywo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8d673633-da6a-42f1-f43c-7c2f4fe62edd"
      },
      "source": [
        "account_value_sb3=test(start_date = TEST_START_DATE, \n",
        "     end_date = TEST_END_DATE,\n",
        "     ticker_list = config.DOW_30_TICKER, \n",
        "     data_source = 'yahoofinance',\n",
        "     time_interval= '1D', \n",
        "     technical_indicator_list= TECHNICAL_INDICATORS_LIST, \n",
        "     drl_lib='stable_baselines3', \n",
        "     env=env, \n",
        "     agent='ppo', \n",
        "     cwd='./test_ppo.zip')"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (7620, 9)\n",
            "Clean data for AAPL\n",
            "Data clean for AAPL is finished.\n",
            "Clean data for AXP\n",
            "Data clean for AXP is finished.\n",
            "Clean data for BA\n",
            "Data clean for BA is finished.\n",
            "Clean data for CAT\n",
            "Data clean for CAT is finished.\n",
            "Clean data for CSCO\n",
            "Data clean for CSCO is finished.\n",
            "Clean data for CVX\n",
            "Data clean for CVX is finished.\n",
            "Clean data for DD\n",
            "Data clean for DD is finished.\n",
            "Clean data for DIS\n",
            "Data clean for DIS is finished.\n",
            "Clean data for GS\n",
            "Data clean for GS is finished.\n",
            "Clean data for HD\n",
            "Data clean for HD is finished.\n",
            "Clean data for IBM\n",
            "Data clean for IBM is finished.\n",
            "Clean data for INTC\n",
            "Data clean for INTC is finished.\n",
            "Clean data for JNJ\n",
            "Data clean for JNJ is finished.\n",
            "Clean data for JPM\n",
            "Data clean for JPM is finished.\n",
            "Clean data for KO\n",
            "Data clean for KO is finished.\n",
            "Clean data for MCD\n",
            "Data clean for MCD is finished.\n",
            "Clean data for MMM\n",
            "Data clean for MMM is finished.\n",
            "Clean data for MRK\n",
            "Data clean for MRK is finished.\n",
            "Clean data for MSFT\n",
            "Data clean for MSFT is finished.\n",
            "Clean data for NKE\n",
            "Data clean for NKE is finished.\n",
            "Clean data for PFE\n",
            "Data clean for PFE is finished.\n",
            "Clean data for PG\n",
            "Data clean for PG is finished.\n",
            "Clean data for RTX\n",
            "Data clean for RTX is finished.\n",
            "Clean data for TRV\n",
            "Data clean for TRV is finished.\n",
            "Clean data for UNH\n",
            "Data clean for UNH is finished.\n",
            "Clean data for V\n",
            "Data clean for V is finished.\n",
            "Clean data for VZ\n",
            "Data clean for VZ is finished.\n",
            "Clean data for WBA\n",
            "Data clean for WBA is finished.\n",
            "Clean data for WMT\n",
            "Data clean for WMT is finished.\n",
            "Clean data for XOM\n",
            "Data clean for XOM is finished.\n",
            "Data clean all finished!\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (254, 9)\n",
            "Clean data for ^VIX\n",
            "Data clean for ^VIX is finished.\n",
            "Data clean all finished!\n",
            "['AAPL' 'AXP' 'BA' 'CAT' 'CSCO' 'CVX' 'DD' 'DIS' 'GS' 'HD' 'IBM' 'INTC'\n",
            " 'JNJ' 'JPM' 'KO' 'MCD' 'MMM' 'MRK' 'MSFT' 'NKE' 'PFE' 'PG' 'RTX' 'TRV'\n",
            " 'UNH' 'V' 'VZ' 'WBA' 'WMT' 'XOM']\n",
            "Successfully transformed into array\n",
            "price_array:  255\n",
            "Successfully load model ./test_ppo.zip\n",
            "episode_return 1.2563062318092366\n",
            "Test Finished!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9WcLm8nYsxeU",
        "outputId": "4b2b51f8-6474-4f36-d24e-614bc2c163bc"
      },
      "source": [
        "len(account_value_sb3)"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "255"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FC0d-lUvj0Zk"
      },
      "source": [
        "### Plot"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uWeh-OTc0i2o",
        "outputId": "b74b126f-f745-4a2d-d8e6-2a7c863b0fdf"
      },
      "source": [
        "baseline_df =  DataProcessor('yahoofinance').download_data(ticker_list = [\"^DJI\"], \n",
        "                                                            start_date = TEST_START_DATE, \n",
        "                                                            end_date = TEST_END_DATE, \n",
        "                                                            time_interval = \"1D\")\n",
        "stats = backtest_stats(baseline_df, value_col_name = 'close')\n"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (254, 9)\n",
            "Annual return          0.302105\n",
            "Cumulative returns     0.304836\n",
            "Annual volatility      0.143039\n",
            "Sharpe ratio           1.925145\n",
            "Calmar ratio           3.382743\n",
            "Stability              0.935738\n",
            "Max drawdown          -0.089308\n",
            "Omega ratio            1.381054\n",
            "Sortino ratio          2.847623\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.074665\n",
            "Daily value at risk   -0.016928\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bMCujva5s5Mc"
      },
      "source": [
        "account_value_sb3 = pd.DataFrame({'date':baseline_df.date,'account_value':account_value_sb3[0:len(account_value_sb3)-1]})"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ifAM8Zgxuy4x",
        "outputId": "54fbb5ec-93d2-4c36-db36-c91c6db2e32a"
      },
      "source": [
        "perf_stats_all = backtest_stats(account_value=account_value_sb3)\n",
        "perf_stats_all = pd.DataFrame(perf_stats_all)"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Annual return          0.254258\n",
            "Cumulative returns     0.256515\n",
            "Annual volatility      0.185498\n",
            "Sharpe ratio           1.318909\n",
            "Calmar ratio           2.670679\n",
            "Stability              0.879340\n",
            "Max drawdown          -0.095203\n",
            "Omega ratio            1.248198\n",
            "Sortino ratio          1.996820\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.066012\n",
            "Daily value at risk   -0.022400\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "YS8-OOGMtIRA",
        "outputId": "ccac2a0b-a39c-42a4-f727-05e342b4a3e3"
      },
      "source": [
        "account_value_sb3.tail()"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>account_value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>249</th>\n",
              "      <td>2021-07-29</td>\n",
              "      <td>1.273393e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>250</th>\n",
              "      <td>2021-07-30</td>\n",
              "      <td>1.262119e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>251</th>\n",
              "      <td>2021-08-02</td>\n",
              "      <td>1.256096e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>252</th>\n",
              "      <td>2021-08-03</td>\n",
              "      <td>1.266162e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>253</th>\n",
              "      <td>2021-08-04</td>\n",
              "      <td>1.256515e+06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           date  account_value\n",
              "249  2021-07-29   1.273393e+06\n",
              "250  2021-07-30   1.262119e+06\n",
              "251  2021-08-02   1.256096e+06\n",
              "252  2021-08-03   1.266162e+06\n",
              "253  2021-08-04   1.256515e+06"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WtSWdl57kDlx",
        "outputId": "2dc4bb59-89eb-4f2e-d0f8-2d20026a4b51"
      },
      "source": [
        "print(\"==============Compare to DJIA===========\")\n",
        "#%matplotlib inline\n",
        "# S&P 500: ^GSPC\n",
        "# Dow Jones Index: ^DJI\n",
        "# NASDAQ 100: ^NDX\n",
        "backtest_plot(account_value_sb3, \n",
        "             baseline_ticker = '^DJI', \n",
        "             baseline_start = account_value_sb3.loc[0,'date'],\n",
        "             baseline_end = account_value_sb3.loc[len(account_value_sb3)-1,'date'])"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "==============Compare to DJIA===========\n",
            "\r[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (253, 8)\n",
            "<IPython.core.display.HTML object>\n",
            "<IPython.core.display.HTML object>\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyfolio/tears.py:907: UserWarning:\n",
            "\n",
            "Passed returns do not overlap with anyinteresting times.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "<Figure size 1008x5184 with 13 Axes>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6oZXzGdRqsbu"
      },
      "source": [
        "## Plotly"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "LqiDBb03r4EG",
        "outputId": "51466f10-2e4a-4993-aa7d-933d43b9e616"
      },
      "source": [
        "baseline_df"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>open</th>\n",
              "      <th>high</th>\n",
              "      <th>low</th>\n",
              "      <th>close</th>\n",
              "      <th>adjcp</th>\n",
              "      <th>volume</th>\n",
              "      <th>tic</th>\n",
              "      <th>day</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-08-03</td>\n",
              "      <td>26542.320312</td>\n",
              "      <td>26707.259766</td>\n",
              "      <td>26534.380859</td>\n",
              "      <td>26664.400391</td>\n",
              "      <td>26664.400391</td>\n",
              "      <td>439380000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-08-04</td>\n",
              "      <td>26664.609375</td>\n",
              "      <td>26832.720703</td>\n",
              "      <td>26597.820312</td>\n",
              "      <td>26828.470703</td>\n",
              "      <td>26828.470703</td>\n",
              "      <td>346350000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-08-05</td>\n",
              "      <td>26924.779297</td>\n",
              "      <td>27221.669922</td>\n",
              "      <td>26924.779297</td>\n",
              "      <td>27201.519531</td>\n",
              "      <td>27201.519531</td>\n",
              "      <td>366000000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-08-06</td>\n",
              "      <td>27170.820312</td>\n",
              "      <td>27394.099609</td>\n",
              "      <td>27145.250000</td>\n",
              "      <td>27386.980469</td>\n",
              "      <td>27386.980469</td>\n",
              "      <td>322460000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-08-07</td>\n",
              "      <td>27321.679688</td>\n",
              "      <td>27456.240234</td>\n",
              "      <td>27223.550781</td>\n",
              "      <td>27433.480469</td>\n",
              "      <td>27433.480469</td>\n",
              "      <td>321170000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>249</th>\n",
              "      <td>2021-07-29</td>\n",
              "      <td>34985.988281</td>\n",
              "      <td>35171.519531</td>\n",
              "      <td>34985.988281</td>\n",
              "      <td>35084.531250</td>\n",
              "      <td>35084.531250</td>\n",
              "      <td>222680000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>250</th>\n",
              "      <td>2021-07-30</td>\n",
              "      <td>35013.261719</td>\n",
              "      <td>35106.300781</td>\n",
              "      <td>34871.128906</td>\n",
              "      <td>34935.468750</td>\n",
              "      <td>34935.468750</td>\n",
              "      <td>276410000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>251</th>\n",
              "      <td>2021-08-02</td>\n",
              "      <td>34968.558594</td>\n",
              "      <td>35192.109375</td>\n",
              "      <td>34808.199219</td>\n",
              "      <td>34838.160156</td>\n",
              "      <td>34838.160156</td>\n",
              "      <td>238620000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>252</th>\n",
              "      <td>2021-08-03</td>\n",
              "      <td>34864.101562</td>\n",
              "      <td>35120.941406</td>\n",
              "      <td>34714.328125</td>\n",
              "      <td>35116.398438</td>\n",
              "      <td>35116.398438</td>\n",
              "      <td>259820000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>253</th>\n",
              "      <td>2021-08-04</td>\n",
              "      <td>35047.441406</td>\n",
              "      <td>35047.441406</td>\n",
              "      <td>34790.621094</td>\n",
              "      <td>34792.671875</td>\n",
              "      <td>34792.671875</td>\n",
              "      <td>245300000</td>\n",
              "      <td>^DJI</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>254 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           date          open          high  ...     volume   tic  day\n",
              "0    2020-08-03  26542.320312  26707.259766  ...  439380000  ^DJI    0\n",
              "1    2020-08-04  26664.609375  26832.720703  ...  346350000  ^DJI    1\n",
              "2    2020-08-05  26924.779297  27221.669922  ...  366000000  ^DJI    2\n",
              "3    2020-08-06  27170.820312  27394.099609  ...  322460000  ^DJI    3\n",
              "4    2020-08-07  27321.679688  27456.240234  ...  321170000  ^DJI    4\n",
              "..          ...           ...           ...  ...        ...   ...  ...\n",
              "249  2021-07-29  34985.988281  35171.519531  ...  222680000  ^DJI    3\n",
              "250  2021-07-30  35013.261719  35106.300781  ...  276410000  ^DJI    4\n",
              "251  2021-08-02  34968.558594  35192.109375  ...  238620000  ^DJI    0\n",
              "252  2021-08-03  34864.101562  35120.941406  ...  259820000  ^DJI    1\n",
              "253  2021-08-04  35047.441406  35047.441406  ...  245300000  ^DJI    2\n",
              "\n",
              "[254 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ae2RzWwqgJ5"
      },
      "source": [
        "from datetime import datetime as dt\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import plotly\n",
        "import plotly.graph_objs as go"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "pcYqpqJzquki",
        "outputId": "3ec6bb03-91a4-4365-dab6-b2d9c0e4746c"
      },
      "source": [
        "account_value_rllib.head()"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>account_value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-08-03</td>\n",
              "      <td>1.000000e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-08-04</td>\n",
              "      <td>9.998989e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-08-05</td>\n",
              "      <td>1.001804e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-08-06</td>\n",
              "      <td>1.001982e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-08-07</td>\n",
              "      <td>1.004149e+06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         date  account_value\n",
              "0  2020-08-03   1.000000e+06\n",
              "1  2020-08-04   9.998989e+05\n",
              "2  2020-08-05   1.001804e+06\n",
              "3  2020-08-06   1.001982e+06\n",
              "4  2020-08-07   1.004149e+06"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9M8G491ewZul"
      },
      "source": [
        "daily_return = account_value_sb3.copy()\n",
        "daily_return['sb3_return'] = account_value_sb3.account_value.pct_change()\n",
        "daily_return['erl_return'] = account_value_erl.account_value.pct_change()\n",
        "daily_return['rllib_return'] = account_value_rllib.account_value.pct_change()\n",
        "daily_return['djia_return'] = baseline_df.adjcp.pct_change()\n"
      ],
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "jgrNgSaKwz-y",
        "outputId": "ae4d24ab-1fc8-4825-dd73-0e03351e0f18"
      },
      "source": [
        "daily_return.head()"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>account_value</th>\n",
              "      <th>sb3_return</th>\n",
              "      <th>erl_return</th>\n",
              "      <th>rllib_return</th>\n",
              "      <th>djia_return</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-08-03</td>\n",
              "      <td>1.000000e+06</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-08-04</td>\n",
              "      <td>9.998347e+05</td>\n",
              "      <td>-0.000165</td>\n",
              "      <td>-0.000065</td>\n",
              "      <td>-0.000101</td>\n",
              "      <td>0.006153</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-08-05</td>\n",
              "      <td>1.002510e+06</td>\n",
              "      <td>0.002676</td>\n",
              "      <td>0.000870</td>\n",
              "      <td>0.001905</td>\n",
              "      <td>0.013905</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-08-06</td>\n",
              "      <td>1.002955e+06</td>\n",
              "      <td>0.000444</td>\n",
              "      <td>0.000770</td>\n",
              "      <td>0.000178</td>\n",
              "      <td>0.006818</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-08-07</td>\n",
              "      <td>1.002987e+06</td>\n",
              "      <td>0.000032</td>\n",
              "      <td>0.001084</td>\n",
              "      <td>0.002163</td>\n",
              "      <td>0.001698</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         date  account_value  sb3_return  erl_return  rllib_return  djia_return\n",
              "0  2020-08-03   1.000000e+06         NaN         NaN           NaN          NaN\n",
              "1  2020-08-04   9.998347e+05   -0.000165   -0.000065     -0.000101     0.006153\n",
              "2  2020-08-05   1.002510e+06    0.002676    0.000870      0.001905     0.013905\n",
              "3  2020-08-06   1.002955e+06    0.000444    0.000770      0.000178     0.006818\n",
              "4  2020-08-07   1.002987e+06    0.000032    0.001084      0.002163     0.001698"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wH2bH3u4w9Gx"
      },
      "source": [
        "daily_return.to_csv('daily_return_erl_sb3_rllib.csv',index=False)\n",
        "#daily_return = pd.read_csv('daily_return_erl_sb3_rllib.csv')"
      ],
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IFs_D1YgqxES"
      },
      "source": [
        "rllib_cumpod =(daily_return.rllib_return+1).cumprod()-1\n",
        "sb3_cumpod =(daily_return.sb3_return+1).cumprod()-1\n",
        "erl_cumpod =(daily_return.erl_return+1).cumprod()-1\n",
        "dji_cumpod =(daily_return.djia_return+1).cumprod()-1"
      ],
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7GD2ubi-wSsq"
      },
      "source": [
        ""
      ],
      "execution_count": 75,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yr5AOj5DqisL"
      },
      "source": [
        "time_ind = pd.Series(daily_return.date)"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HzLZbaLcqith"
      },
      "source": [
        "trace0_portfolio = go.Scatter(x = time_ind, y = rllib_cumpod, mode = 'lines', name = 'RLlib')\n",
        "\n",
        "trace1_portfolio = go.Scatter(x = time_ind, y = dji_cumpod, mode = 'lines', name = 'DJIA')\n",
        "trace2_portfolio = go.Scatter(x = time_ind, y = sb3_cumpod, mode = 'lines', name = 'Stablebaselines3')\n",
        "trace3_portfolio = go.Scatter(x = time_ind, y = erl_cumpod, mode = 'lines', name = 'ElegantRL')\n",
        "#trace4_portfolio = go.Scatter(x = time_ind, y = addpg_cumpod, mode = 'lines', name = 'Adaptive-DDPG')\n",
        "#trace5_portfolio = go.Scatter(x = time_ind, y = min_cumpod, mode = 'lines', name = 'Min-Variance')\n",
        "\n",
        "#trace4 = go.Scatter(x = time_ind, y = addpg_cumpod, mode = 'lines', name = 'Adaptive-DDPG')\n",
        "\n",
        "#trace2 = go.Scatter(x = time_ind, y = portfolio_cost_minv, mode = 'lines', name = 'Min-Variance')\n",
        "#trace3 = go.Scatter(x = time_ind, y = spx_value, mode = 'lines', name = 'SPX')"
      ],
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gFGHOl0SrMdg"
      },
      "source": [
        "fig = go.Figure()\n",
        "fig.add_trace(trace0_portfolio)\n",
        "\n",
        "#fig.add_trace(trace1_portfolio)\n",
        "\n",
        "#fig.add_trace(trace2_portfolio)\n",
        "\n",
        "\n",
        "\n",
        "fig.update_layout(\n",
        "    legend=dict(\n",
        "        x=0,\n",
        "        y=1,\n",
        "        traceorder=\"normal\",\n",
        "        font=dict(\n",
        "            family=\"sans-serif\",\n",
        "            size=15,\n",
        "            color=\"black\"\n",
        "        ),\n",
        "        bgcolor=\"White\",\n",
        "        bordercolor=\"white\",\n",
        "        borderwidth=2\n",
        "        \n",
        "    ),\n",
        ")\n",
        "#fig.update_layout(legend_orientation=\"h\")\n",
        "fig.update_layout(title={\n",
        "        #'text': \"Cumulative Return using FinRL\",\n",
        "        'y':0.85,\n",
        "        'x':0.5,\n",
        "        'xanchor': 'center',\n",
        "        'yanchor': 'top'})\n",
        "#with Transaction cost\n",
        "#fig.update_layout(title =  'Quarterly Trade Date')\n",
        "fig.update_layout(\n",
        "#    margin=dict(l=20, r=20, t=20, b=20),\n",
        "\n",
        "    paper_bgcolor='rgba(1,1,0,0)',\n",
        "    plot_bgcolor='rgba(1, 1, 0, 0)',\n",
        "    #xaxis_title=\"Date\",\n",
        "    yaxis_title=\"Cumulative Return\",\n",
        "xaxis={'type': 'date', \n",
        "       'tick0': time_ind[0], \n",
        "        'tickmode': 'linear', \n",
        "       'dtick': 86400000.0 *80}\n",
        "\n",
        ")\n",
        "fig.update_xaxes(showline=True,linecolor='black',showgrid=True, gridwidth=1, gridcolor='LightSteelBlue',mirror=True)\n",
        "fig.update_yaxes(showline=True,linecolor='black',showgrid=True, gridwidth=1, gridcolor='LightSteelBlue',mirror=True)\n",
        "fig.update_yaxes(zeroline=True, zerolinewidth=1, zerolinecolor='LightSteelBlue')\n",
        "\n",
        "fig.show()"
      ],
      "execution_count": 77,
      "outputs": []
    }
  ]
}