Quandl price handler

.gitignore

@@ -30,3 +30,10 @@ pip-log.txt
 
 #Mr Developer
 .mr.developer.cfg
+
+# Textmate (a Mac editor) meta files
+._*
+
+# Hides quandl config file that may contain
+# private information (i.e. quandl api key)
+quandl_conf.py

examples/quandl_futures_backtest.py

@@ -0,0 +1,74 @@
+from decimal import Decimal
+
+from qstrader.backtest.backtest import Backtest
+from qstrader.execution_handler.execution_handler import IBSimulatedExecutionHandler
+from qstrader.portfolio_handler.portfolio_handler import PortfolioHandler
+from qstrader.position_sizer.position_sizer import TestPositionSizer
+from qstrader.price_handler.quandl_price_handler import QuandlPriceHandler
+from qstrader.risk_manager.risk_manager import TestRiskManager
+from qstrader.statistics.statistics import SimpleStatistics
+from qstrader import settings
+from qstrader.strategy.strategy import TestStrategy
+try:
+		import Queue as queue
+except ImportError:
+		import queue
+
+def main(config, testing=False):
+    # see: https://www.quandl.com/collections/futures
+    tickers = [ "CHRIS/CME_ES1",    # /ES (front month)
+            #    "CHRIS/CME_NQ1",   # /NQ (front month)
+            #    "CHRIS/CME_YM1",   # /YM (front month)
+            #    "CHRIS/ICE_TF1",   # /TF (front month)
+            #    "CHRIS/CME_GC1",   # /GC (front month)
+            #    "CHRIS/CME_SI1",   # /SI (front month)
+              ]
+
+    # Set up variables needed for backtest
+    events_queue = queue.Queue()
+    csv_dir = config.CSV_DATA_DIR
+    initial_equity = Decimal("500000.00")
+    heartbeat = 0.0
+    max_iters = 10000000000
+
+    # Use Historic CSV Price Handler
+    price_handler = QuandlPriceHandler(
+        csv_dir, events_queue, tickers, config=config
+    )
+
+    # Use the Test Strategy
+    strategy = TestStrategy( tickers, events_queue )
+
+    # Use an example Position Sizer
+    position_sizer = TestPositionSizer()
+
+    # Use an example Risk Manager
+    risk_manager = TestRiskManager()
+
+    # Use the default Portfolio Handler
+    portfolio_handler = PortfolioHandler(
+        initial_equity, events_queue, price_handler,
+        position_sizer, risk_manager
+    )
+
+    # Use a simulated IB Execution Handler
+    execution_handler = IBSimulatedExecutionHandler(
+        events_queue, price_handler
+    )
+
+    # Use the default Statistics
+    statistics = SimpleStatistics(portfolio_handler)
+
+    # Set up the backtest
+    backtest = Backtest(
+        tickers, price_handler,
+        strategy, portfolio_handler,
+        execution_handler,
+        position_sizer, risk_manager,
+        statistics,
+        initial_equity
+    )
+    backtest.simulate_trading(testing=testing)
+
+if __name__ == "__main__":
+    main(settings.DEFAULT, testing=False)

qstrader/price_handler/quandl_price_handler.py

@@ -0,0 +1,170 @@
+import datetime
+from decimal import Decimal, getcontext, ROUND_HALF_DOWN
+import os, os.path
+
+import pandas as pd
+
+from qstrader.event.event import BarEvent
+from qstrader.price_handler.price_handler import PriceHandler
+from qstrader.settings import DEFAULT as settings
+
+import quandl
+
+class QuandlPriceHandler(PriceHandler):
+    """
+    QuandlBarPriceHandler is designed to read CSV files of
+    Quandl daily Open-High-Low-Close-Volume (OHLCV) data
+    for each requested financial instrument and stream those to
+    the provided events queue as BarEvents.
+
+    Note: quandl.get() already retrieves data in Pandas DataFrames.  So instead
+          of storing those as CSV files, just use the the DataFrame directly.
+          Later, cache the data as a CSV or sqlite db for better
+          performance.
+    """
+    def __init__(self, csv_dir, events_queue, init_tickers=None,
+                 config=settings):
+        """
+        Takes the CSV directory, the events queue and a possible
+        list of initial ticker (quandl) symbols then creates an (optional)
+        list of ticker subscriptions and associated prices.
+        """
+        # Globally set the Quandl Key and other quandl config parms
+        # (see comment in quandl_conf.py)
+        #
+        # Note:
+        # quandl.get() also supports a parm called api_key.  Specifying the
+        # api_key and setting the key directly in the ApiConfig class
+        # effectively are the same thing.  The key only needs to be specified
+        # here once, as opposed to on each API call.
+        quandl.ApiConfig.api_key = config.QUANDL_API_KEY
+        quandl.ApiConfig.api_base = config.QUANDL_API_BASE
+        quandl.ApiConfig.api_version = config.QUANDL_API_VERSION
+        quandl.ApiConfig.page_limit = config.QUANDL_PAGE_LIMIT
+
+
+        self.type = "BAR_HANDLER"
+        self.csv_dir = csv_dir
+        self.events_queue = events_queue
+        self.continue_backtest = True
+        self.tickers = {}
+        self.tickers_data = {}
+        if init_tickers is not None:
+            for ticker in init_tickers:
+                self.subscribe_ticker(ticker)
+        self.bar_stream = self._merge_sort_ticker_data()
+
+    def _open_ticker_price_csv(self, ticker):
+        """
+        Opens the CSV files containing the equities ticks from
+        the specified CSV data directory, converting them into
+        them into a pandas DataFrame, stored in a dictionary.
+        """
+        # TBD: for now just get straight from quandl
+        # TBD: Input start_date from user.  For now just hard code it.
+        self.tickers_data[ticker] = quandl.get(ticker,
+                                               start_date="2016-05-01")
+        self.tickers_data[ticker]["Ticker"] = ticker
+        """
+        ticker_path = os.path.join(self.csv_dir, "%s.csv" % ticker)
+        self.tickers_data[ticker] = pd.io.parsers.read_csv(
+            ticker_path, header=0, parse_dates=True,
+            index_col=0, names=(
+                "Date", "Open", "High", "Low",
+                "Last", "Change", "Settle", "Volume"
+            )
+        )
+        self.tickers_data[ticker]["Ticker"] = ticker
+        """
+
+    def _merge_sort_ticker_data(self):
+        """
+        Concatenates all of the separate equities DataFrames
+        into a single DataFrame that is time ordered, allowing tick
+        data events to be added to the queue in a chronological fashion.
+
+        Note that this is an idealised situation, utilised solely for
+        backtesting. In live trading ticks may arrive "out of order".
+        """
+        return pd.concat(
+            self.tickers_data.values()
+        ).sort_index().iterrows()
+
+    def subscribe_ticker(self, ticker):
+        """
+        Subscribes the price handler to a new ticker symbol.
+        """
+        if ticker not in self.tickers:
+            try:
+                self._open_ticker_price_csv(ticker)
+                dft = self.tickers_data[ticker]
+                row0 = dft.iloc[0]
+                ticker_prices = {
+                    "last": Decimal(
+                        str(row0["Last"])
+                    ).quantize(Decimal("0.00001")),
+                    "settle": Decimal(
+                        str(row0["Settle"])
+                    ).quantize(Decimal("0.00001")),
+                    "timestamp": dft.index[0]
+                }
+                self.tickers[ticker] = ticker_prices
+            except OSError:
+                print(
+                    "Could not subscribe ticker %s " \
+                    "as no data CSV found for pricing." % ticker
+                )
+        else:
+            print(
+                "Could not subscribe ticker %s " \
+                "as is already subscribed." % ticker
+            )
+
+    def get_last_close(self, ticker):
+        """
+        Returns the most recent actual (unadjusted) closing price.
+        """
+        if ticker in self.tickers:
+            close_price = self.tickers[ticker]["last"]
+            return close_price
+        else:
+            print(
+                "Close price for ticker %s is not " \
+                "available from the QuandlPriceHandler."
+            )
+            return None
+
+    def stream_next_bar(self):
+        """
+        Place the next BarEvent onto the event queue.
+        """
+        try:
+            index, row = next(self.bar_stream)
+        except StopIteration:
+            self.continue_backtest = False
+            return
+
+        # Obtain all elements of the bar from the dataframe
+        getcontext().rounding = ROUND_HALF_DOWN
+        ticker = row["Ticker"]
+        open_price = Decimal(str(row["Open"])).quantize(Decimal("0.00001"))
+        high_price = Decimal(str(row["High"])).quantize(Decimal("0.00001"))
+        low_price = Decimal(str(row["Low"])).quantize(Decimal("0.00001"))
+        last_price = Decimal(str(row["Last"])).quantize(Decimal("0.00001"))
+        settle_price = Decimal(str(row["Settle"])).quantize(Decimal("0.00001"))
+        volume = int(row["Volume"])
+
+        # Create decimalised prices for
+        # closing price and adjusted closing price
+        self.tickers[ticker]["last"] = last_price
+        self.tickers[ticker]["settle"] = settle_price
+        self.tickers[ticker]["timestamp"] = index
+
+        # Create the tick event for the queue
+        period = 86400  # Seconds in a day
+        bev = BarEvent(
+            ticker, index, period, open_price,
+            high_price, low_price, last_price,
+            volume, settle_price
+        )
+        self.events_queue.put(bev)

qstrader/quandl_conf.py.example

@@ -0,0 +1,33 @@
+import quandl
+
+# Quandl Key
+# see https://www.quandl.com/blog/getting-started-with-the-quandl-api
+# see https://www.quandl.com/docs/api#api-keys
+#
+# Requests without an API key (anonymous requests) are accepted, but subject to
+# strict rate limits. This allows users to test the API without signing up for
+# a Quandl account.
+#
+# Anonymous requests are subject to a limit of 50 calls per day.
+# Anonymous requests for premium databases are not accepted.
+# Authenticated users have a limit of 2,000 calls per 10 minutes, and a limit
+# of 50,000 calls per day.  Premium data subscribers have a limit of 5,000
+# calls per 10 minutes, and a limit of 720,000 calls per day.
+#
+# If a valid API key is not used, some datatables will default
+# to returning sample data. If you are not receiving all expected data,
+# please double check your API key.
+# i.e. QUANDL_API_KEY="<enter your free quandl key here>"
+QUANDL_API_KEY = quandl.ApiConfig.api_key            # Default, No key
+
+# Specifies the Quandl API version to use
+# i.e. QUANDL_API_VERSION = '2015-04-09'
+QUANDL_API_VERSION = quandl.ApiConfig.api_version    # uses default
+
+# API base
+# i.e. QUANDL_API_BASE = 'https://www.quandl.com/api/v3'
+QUANDL_API_BASE = quandl.ApiConfig.api_base          # uses default
+
+# Page Limit
+# i.e. QUANDL_PAGE_LIMIT = 100
+QUANDL_PAGE_LIMIT = quandl.ApiConfig.page_limit      # uses default

qstrader/settings.py

@@ -1,10 +1,9 @@
 import os
 import warnings
-
+import quandl
 
 ENV_VAR_ROOT = 'QSTRADER'
 
-
 def get_info(key, default_value=None):
     """Returns a value (url, login, password)
     using either default_value or using environment variable"""
@@ -23,6 +22,21 @@ class SettingsDefault(object):
     _CSV_DATA_DIR = "~/data"
     _OUTPUT_DIR = "~/qstrader"
 
+    # Quandl config is separated out to a config file to protect private keys,
+    # but left in setting.py to keep all settings organized in one place.
+    try:
+       import quandl_conf
+       _QUANDL_KEY = quandl_conf.QUANDL_API_KEY
+       _QUANDL_BASE = quandl_conf.QUANDL_API_BASE
+       _QUANDL_VERSION = quandl_conf.QUANDL_API_VERSION
+       _QUANDL_PAGE_LIMIT = quandl_conf.QUANDL_PAGE_LIMIT
+    except ImportError:
+       # quandl_conf.py is missing.  Grab quandl defaults.
+       _QUANDL_KEY = quandl.ApiConfig.api_key
+       _QUANDL_BASE = quandl.ApiConfig.api_base
+       _QUANDL_VERSION = quandl.ApiConfig.api_version
+       _QUANDL_PAGE_LIMIT = quandl.ApiConfig.page_limit
+
     @property
     def CSV_DATA_DIR(self):
         return get_info("CSV_DATA_DIR", os.path.expanduser(self._CSV_DATA_DIR))
@@ -31,10 +45,35 @@ def CSV_DATA_DIR(self):
     def OUTPUT_DIR(self):
         return get_info("OUTPUT_DIR", os.path.expanduser(self._CSV_DATA_DIR))
 
+    #**************************************************************************
+    # Quandl config data
+    # Note: Do not use get_info because default_values could be 'None'
+    #**************************************************************************
+    @property
+    def QUANDL_API_KEY(self):
+        return self._QUANDL_KEY
+
+    @property
+    def QUANDL_API_BASE(self):
+        return self._QUANDL_BASE
+
+    @property
+    def QUANDL_API_VERSION(self):
+        return self._QUANDL_VERSION
+
+    @property
+    def QUANDL_PAGE_LIMIT(self):
+        return self._QUANDL_PAGE_LIMIT
+
+
 
 class SettingsTest(SettingsDefault):
     _CSV_DATA_DIR = "data"
     _OUTPUT_DIR = "out"
+    _QUANDL_KEY = quandl.ApiConfig.api_key
+    _QUANDL_BASE = quandl.ApiConfig.api_base
+    _QUANDL_VERSION = quandl.ApiConfig.api_version
+    _QUANDL_PAGE_LIMIT = quandl.ApiConfig.page_limit
 
 
 DEFAULT = SettingsDefault()

qstrader/strategy/strategy.py

@@ -43,7 +43,7 @@ def __init__(self, tickers, events_queue):
 
     def calculate_signals(self, event):
         ticker = self.tickers[0]
-        if event.type == 'TICK' and event.ticker == ticker:
+        if event.ticker == ticker:
             if self.ticks % 5 == 0:
                 if not self.invested:
                     signal = SignalEvent(ticker, "BOT")

tests/test_examples.py

@@ -12,7 +12,7 @@
 import examples.sp500tr_buy_and_hold_backtest
 import examples.mac_backtest
 import examples.strategy_backtest
-
+import examples.quandl_futures_backtest
 
 class TestExamples(unittest.TestCase):
     """
@@ -41,3 +41,9 @@ def test_strategy_backtest(self):
         Test strategy_backtest
         """
         examples.strategy_backtest.main(self.config, testing=True)
+
+    def test_quandl_backtest(self):
+        """
+        Test quandl_futures_backtest
+        """
+        examples.quandl_futures_backtest.main(self.config, testing=True)