Change retries remove historical data from analyzer

app/analysis.py

@@ -2,7 +2,7 @@
 """
 
 import math
-from datetime import datetime, timedelta, timezone
+from datetime import datetime
 
 import structlog
 import pandas
@@ -53,32 +53,6 @@ def __convert_to_dataframe(self, historical_data):
         return dataframe
 
 
-    def get_historical_data(self, market_pair, exchange, time_unit, max_days=100):
-        """Fetches the historical data
-
-        Args:
-            market_pair (str): Contains the symbol pair to operate on i.e. BURST/BTC
-            exchange (str): Contains the exchange to fetch the historical data from.
-            time_unit (str): A string specifying the ccxt time unit i.e. 5m or 1d.
-            max_days (int, optional): Defaults to 100. Maximum number of days to fetch data for.
-
-        Returns:
-            list: Contains a list of lists which contain timestamp, open, high, low, close, volume.
-        """
-
-        # The data_start_date timestamp must be in milliseconds hence * 1000.
-        data_start_date = datetime.now() - timedelta(days=max_days)
-        data_start_date = int(data_start_date.replace(tzinfo=timezone.utc).timestamp() * 1000)
-        historical_data = self.__exchange_interface.get_historical_data(
-            market_pair=market_pair,
-            exchange=exchange,
-            time_unit=time_unit,
-            start_date=data_start_date
-        )
-
-        return historical_data
-
-
     def analyze_macd(self, historial_data, hot_thresh=None, cold_thresh=None, all_data=False):
         """Performs a macd analysis on the historical data
 

app/behaviours/default.py

@@ -29,6 +29,7 @@ def __init__(self, behaviour_config, exchange_interface, strategy_analyzer, noti
         self.strategy_analyzer = strategy_analyzer
         self.notifier = notifier
 
+
     def run(self, market_pairs):
         """The behaviour entrypoint
 
@@ -46,6 +47,7 @@ def run(self, market_pairs):
 
         self.__test_strategies(market_data)
 
+
     def __test_strategies(self, market_data):
         """Test the strategies and perform notifications as required
 
@@ -57,13 +59,13 @@ def __test_strategies(self, market_data):
             for market_pair in market_data[exchange]:
 
                 try:
-                    one_day_historical_data = self.strategy_analyzer.get_historical_data(
+                    one_day_historical_data = self.exchange_interface.get_historical_data(
                         market_data[exchange][market_pair]['symbol'],
                         exchange,
                         '1d'
                     )
 
-                    five_minute_historical_data = self.strategy_analyzer.get_historical_data(
+                    five_minute_historical_data = self.exchange_interface.get_historical_data(
                         market_data[exchange][market_pair]['symbol'],
                         exchange,
                         '5m'

app/behaviours/simple_bot.py

@@ -3,9 +3,7 @@
 
 from datetime import datetime, timedelta
 
-import ccxt
 import structlog
-from tenacity import retry, retry_if_exception_type, stop_after_attempt
 
 class SimpleBotBehaviour():
     """Simple trading bot.
@@ -56,9 +54,10 @@ def run(self, market_pairs):
             analyzed_data[exchange] = {}
 
             for market_pair in markets:
-                historical_data = self.__get_historical_data(
+                historical_data = self.exchange_interface.get_historical_data(
                     market_data[exchange][market_pair]['symbol'],
-                    exchange
+                    exchange,
+                    self.behaviour_config['analysis_timeframe']
                 )
 
                 strategy_result = self.__run_strategy(historical_data)
@@ -122,30 +121,6 @@ def run(self, market_pairs):
         self.logger.debug(current_holdings)
 
 
-    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
-    def __get_historical_data(self, symbol, exchange):
-        """Get the historical data for a given pair.
-
-        Decorators:
-            retry
-
-        Args:
-            symbol (str): The symbol pair that we want to get historical data for.
-            exchange (str): The exchange id of the exchange we want the historical data for.
-
-        Returns:
-            list: A matrix of historical OHLCV
-        """
-
-        historical_data = self.strategy_analyzer.get_historical_data(
-            symbol,
-            exchange,
-            self.behaviour_config['analysis_timeframe']
-        )
-
-        return historical_data
-
-
     def __run_strategy(self, historical_data):
         """Run the selected analyzer over the historical data
 
@@ -203,12 +178,8 @@ def __run_strategy(self, historical_data):
         return result
 
 
-    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def __reconcile_open_orders(self):
         """Cancels any orders that have been open for too long.
-
-        Decorators:
-            retry
         """
         open_orders = self.exchange_interface.get_open_orders()
 
@@ -230,13 +201,9 @@ def __reconcile_open_orders(self):
                         )
 
 
-    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def buy(self, base_symbol, quote_symbol, market_pair, exchange, current_holdings):
         """Buy a base currency with a quote currency.
 
-        Decorators:
-            retry
-
         Args:
             base_symbol (str): The symbol for the base currency (currency being bought).
             quote_symbol (str): The symbol for the quote currency (currency being sold).
@@ -317,13 +284,9 @@ def buy(self, base_symbol, quote_symbol, market_pair, exchange, current_holdings
         self.db_handler.create_transaction(purchase_payload)
 
 
-    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def sell(self, base_symbol, quote_symbol, market_pair, exchange, current_holdings):
         """Sell a base currency for a quote currency.
 
-        Decorators:
-            retry
-
         Args:
             base_symbol (str): The symbol for the base currency (currency being sold).
             quote_symbol (str): The symbol for the quote currency (currency being bought).
@@ -415,12 +378,8 @@ def __get_holdings(self):
         return holdings
 
 
-    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def __create_holdings(self):
         """Query the users account details to populate the crypto holdings database cache.
-
-        Decorators:
-            retry
         """
 
         for exchange in self.exchange_interface.exchanges:
@@ -451,12 +410,8 @@ def __create_holdings(self):
                 self.db_handler.create_holding(holding_payload)
 
 
-    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def __update_holdings(self):
         """Synchronize the database cache with the crypto holdings from the users account.
-
-        Decorators:
-            retry
         """
 
         holdings_table = self.db_handler.read_holdings()

app/exchange.py

@@ -2,9 +2,11 @@
 """
 
 import time
+from datetime import datetime, timedelta, timezone
 
 import ccxt
 import structlog
+from tenacity import retry, retry_if_exception_type, stop_after_attempt
 
 class ExchangeInterface():
     """Interface for performing queries against exchange API's
@@ -59,31 +61,37 @@ def override_exchange_config(self):
             })
 
 
-    def get_historical_data(self, market_pair, exchange, time_unit, start_date):
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
+    def get_historical_data(self, market_pair, exchange, time_unit, start_date=None, max_days=100):
         """Get historical OHLCV for a symbol pair
 
         Args:
             market_pair (str): Contains the symbol pair to operate on i.e. BURST/BTC
             exchange (str): Contains the exchange to fetch the historical data from.
             time_unit (str): A string specifying the ccxt time unit i.e. 5m or 1d.
-            start_date (int): Timestamp in milliseconds.
+            start_date (int, optional): Timestamp in milliseconds.
+            max_days (int, optional): Defaults to 100. Maximum number of days to fetch data for
+                if start date is not specified.
 
         Returns:
             list: Contains a list of lists which contain timestamp, open, high, low, close, volume.
         """
 
-        historical_data = []
-        historical_data.append(
-            self.exchanges[exchange].fetch_ohlcv(
-                market_pair,
-                timeframe=time_unit,
-                since=start_date
-            )
+        if not start_date:
+            max_days_date = datetime.now() - timedelta(days=max_days)
+            start_date = int(max_days_date.replace(tzinfo=timezone.utc).timestamp() * 1000)
+
+        historical_data = self.exchanges[exchange].fetch_ohlcv(
+            market_pair,
+            timeframe=time_unit,
+            since=start_date
         )
+
         time.sleep(self.exchanges[exchange].rateLimit / 1000)
-        return historical_data[0]
+        return historical_data
 
 
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def get_account_markets(self, exchange):
         """Get the symbol pairs listed within a users account.
 
@@ -99,6 +107,8 @@ def get_account_markets(self, exchange):
         time.sleep(self.exchanges[exchange].rateLimit / 1000)
         return account_markets
 
+
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def get_markets_for_exchange(self, exchange):
         """Get market data for all symbol pairs listed on the given exchange.
 
@@ -114,6 +124,7 @@ def get_markets_for_exchange(self, exchange):
         return exchange_markets
 
 
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def get_exchange_markets(self):
         """Get market data for all symbol pairs listed on all configured exchanges.
 
@@ -128,6 +139,7 @@ def get_exchange_markets(self):
         return exchange_markets
 
 
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def get_symbol_markets(self, market_pairs):
         """Get market data for specific symbols on all configured exchanges.
 
@@ -150,6 +162,7 @@ def get_symbol_markets(self, market_pairs):
         return symbol_markets
 
 
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def get_order_book(self, market_pair, exchange):
         """Retrieve the order information for a particular symbol pair.
 
@@ -163,6 +176,8 @@ def get_order_book(self, market_pair, exchange):
 
         return self.exchanges[exchange].fetch_order_book(market_pair)
 
+
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def get_open_orders(self):
         """Get the users currently open orders on all configured exchanges.
 
@@ -176,6 +191,8 @@ def get_open_orders(self):
             time.sleep(self.exchanges[exchange].rateLimit / 1000)
         return open_orders
 
+
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def cancel_order(self, exchange, order_id):
         """Cancels an open order on a particular exchange.
 
@@ -187,6 +204,8 @@ def cancel_order(self, exchange, order_id):
         self.exchanges[exchange].cancel_order(order_id)
         time.sleep(self.exchanges[exchange].rateLimit / 1000)
 
+
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def get_quote_symbols(self, exchange):
         """Get a list of quote symbols on an exchange.
 
@@ -205,18 +224,16 @@ def get_quote_symbols(self, exchange):
 
         return quote_symbols
 
+
+    @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
     def get_btc_value(self, exchange, base_symbol, volume):
 
         btc_value = 0
         market_pair = base_symbol + "/BTC"
 
-        try:
-            order_book = self.get_order_book(market_pair, exchange)
-            bid = order_book['bids'][0][0] if order_book['bids'] else None
-            if bid:
-                btc_value = bid * volume
-
-        except ccxt.BaseError:
-            self.logger.warn("Unable to get btc value for %s", base_symbol)
+        order_book = self.get_order_book(market_pair, exchange)
+        bid = order_book['bids'][0][0] if order_book['bids'] else None
+        if bid:
+            btc_value = bid * volume
 
         return btc_value