[WIP] handle binance insufficient funds because of partial fills (ask…

…mike#2450)

* [GB] retry when binance says insufficient funds, see askmike#2405

* [GB] debug log oldOrder to check for partial fills

* ref proper oldOrder var, see askmike#2405

* [docs] document fill object in cancelOrder

* [GB] implement insufficientFunds handling on markets with limitedCancelConfirmation, fix askmike#2405

* [GB] create order after correcting insufficientFunds

* [GB] ref correct market prop

* listen to proper fill event

* [GB] rm debug logging

docs/extending/add_an_exchange.md

@@ -62,21 +62,17 @@ The callback needs to have the parameters of `err` and `portfolio`. Portfolio ne
 
 The callback needs to have the parameters of `err` and `lot`. Lot needs to be an object with `amount` and `purchase` size appropriately for the exchange. In the event that the lot is too small, return 0 to both fields and this will generate a lot size warning in the portfolioManager.
 
-Note: This function is currently optional. If not implemented `portfolioManager` will fallback to basic lot sizing mechanism it uses internally. However exchanges are not all the same in how rounding and lot sizing work, it is recommend to implement this function.
+Note: This function is currently optional. If not implemented Gekko Broker will fallback to basic lot sizing mechanism it uses internally. However exchanges are not all the same in how rounding and lot sizing work, it is recommend to implement this function.
 
 ### buy
 
     this.exchange.buy(amount, price, callback);
 
-*Note that this function is a critical function, retry handlers should abort quickly if attempts to dispatch this to the exchange API fail so we don't post out of date orders to the books.*
-
 ### sell
 
     this.exchange.sell(amount, price, callback);
 
-This should create a buy / sell order at the exchange for [amount] of [asset] at [price] per 1 asset. If you have set `direct` to `true` the price will be `false`. The callback needs to have the parameters `err` and `order`. The order needs to be something that can be fed back to the exchange to see whether the order has been filled or not.
-
-*Note that this function is a critical function, retry handlers should abort quickly if attempts to dispatch this to the exchange API fail so we don't post out of date orders to the books.*
+This should create a buy / sell order at the exchange for [amount] of [asset] at [price] per 1 asset. The callback needs to have the parameters `err` and `order`. The order needs to be some id that can be passed back to `checkOrder`, `getOrder` and `cancelOrder`.
 
 ### getOrder
 
@@ -98,7 +94,7 @@ The order will be something that the manager previously received via the `sell`
 
     this.exchange.cancelOrder(order, callback);
 
-The order will be something that the manager previously received via the `sell` or `buy` methods. The callback should have the parameters `err` and `filled`, `filled` last one should be true if the order was filled before it could be cancelled.
+The order will be something that the manager previously received via the `sell` or `buy` methods. The callback should have the parameters `err`, `filled` and optionally a `fill` object, `filled` should be true if the order was fully filled before it could be cancelled. If the exchange provided how much was filled before the cancel this should be passed in the `fill object` as a `amountFilled` property. If the exchange provided how much of the original amount is still remaining, pass this as the `remaining` property instead.
 
 ### roundPrice
 
@@ -157,6 +153,7 @@ Each exchange *must* provide a `getCapabilities()` static method that returns an
 - `requires`: if gekko supports automatic trading, this is an array of required api credentials gekko needs to pass into the constructor.
 - `forceReorderDelay`: if after canceling an order a new one can't be created straight away since the balance is not updated fast enough, set this to true (only required for exchanges where Gekko can trade).
 - `gekkoBroker`: set this to "0.6.2" for now, it indicates the version of Gekko Broker this wrapper is compatible with.
+- `limitedCancelConfirmation` set to true if (in any case) the wrapper does NOT include partial fill information in the response (which is passed as the cancel callback)
 
 Below is a real-case example how `bistamp` exchange provides its `getCapabilities()` method:
 
@@ -177,7 +174,8 @@ Trader.getCapabilities = function () {
     requires: ['key', 'secret', 'username'],
     fetchTimespan: 60,
     tid: 'tid',
-    gekkoBroker: '0.6.2'
+    gekkoBroker: '0.6.2',
+    limitedCancelConfirmation: false
   };
 }
 ```

exchange/gekkoBroker.js

@@ -134,14 +134,17 @@ class Broker {
     if(!orders[type])
       throw new Error('Unknown order type');
 
-    const order = new orders[type](this.api);
+    const order = new orders[type]({
+      api: this.api,
+      marketConfig: this.marketConfig,
+      capabilities: this.capabilities
+    });
 
     // todo: figure out a smarter generic way
     this.syncPrivateData(() => {
       order.setData({
         balances: this.portfolio.balances,
         ticker: this.ticker,
-        market: this.marketConfig
       });
 
       order.create(side, amount, parameters);

exchange/orders/order.js

@@ -26,7 +26,7 @@ class BaseOrder extends EventEmitter {
 
   submit({side, amount, price, alreadyFilled}) {
     const check = isValidOrder({
-      market: this.data.market,
+      market: this.market,
       api: this.api,
       amount,
       price

exchange/orders/sticky.js

@@ -21,9 +21,12 @@ const BaseOrder = require('./order');
 const states = require('./states');
 
 class StickyOrder extends BaseOrder {
-  constructor(api) {
+  constructor({api, marketConfig, capabilities}) {
     super(api);
 
+    this.market = marketConfig;
+    this.capabilities = capabilities;
+
     // global async lock
     this.sticking = false;
 
@@ -126,7 +129,56 @@ class StickyOrder extends BaseOrder {
     });
   }
 
+  // check if the last order was partially filled
+  // on an exchange that does not pass fill data on cancel
+  // see https://github.com/askmike/gekko/pull/2450
+  handleInsufficientFundsError(err) {
+    if(
+      !err ||
+      err.type !== 'insufficientFunds' ||
+      !this.capabilities.limitedCancelConfirmation ||
+      !this.id
+    ) {
+      return false;
+    }
+
+    const id = this.id;
+
+    setTimeout(
+      () => {
+        this.api.getOrder(id, (innerError, res) => {
+          if(this.handleError(innerError)) {
+            return;
+          }
+
+          const amount = res.amount;
+
+          if(this.orders[id].filled === amount) {
+            // handle original error
+            return this.handleError(err);
+          }
+
+          this.orders[id].filled = amount;
+          this.emit('fill', this.calculateFilled());
+          if(this.calculateFilled() >= this.amount) {
+            return this.filled(this.price);
+          }
+
+          setTimeout(this.createOrder, this.checkInterval);
+        });
+      },
+      this.checkInterval
+    );
+
+    return true;
+  }
+
   handleCreate(err, id) {
+
+    if(this.handleInsufficientFundsError(err)) {
+      return;
+    }
+
     if(this.handleError(err)) {
       return;
     }
@@ -394,7 +446,7 @@ class StickyOrder extends BaseOrder {
 
     this.amount = this.roundAmount(amount - this.calculateFilled());
 
-    if(this.amount < this.data.market.minimalOrder.amount) {
+    if(this.amount < this.market.minimalOrder.amount) {
       if(this.calculateFilled()) {
         // we already filled enough of the order!
         this.filled();

exchange/wrappers/binance.js

@@ -64,6 +64,8 @@ const Trader = function(config) {
     this.fee = 0.1;
     // Set the proper fee asap.
     this.getFee(_.noop);
+
+    this.oldOrder = false;
   }
 };
 
@@ -117,6 +119,10 @@ Trader.prototype.handleResponse = function(funcName, callback) {
         return callback(false, {filled: true});
       }
 
+      if(funcName === 'addOrder' && error.message.includes('Account has insufficient balance')) {
+        error.type = 'insufficientFunds';
+      }
+
       return callback(error);
     }
 
@@ -301,7 +307,6 @@ Trader.prototype.isValidPrice = function(price) {
 }
 
 Trader.prototype.isValidLot = function(price, amount) {
-  console.log('isValidLot', this.market.minimalOrder.order, amount * price >= this.market.minimalOrder.order)
   return amount * price >= this.market.minimalOrder.order;
 }
 
@@ -460,7 +465,12 @@ Trader.prototype.checkOrder = function(order, callback) {
 Trader.prototype.cancelOrder = function(order, callback) {
 
   const cancel = (err, data) => {
+
+    this.oldOrder = order;
+
     if(err) {
+      if(err.message.contains(''))
+
       return callback(err);
     }
 
@@ -492,7 +502,8 @@ Trader.getCapabilities = function() {
     providesFullHistory: true,
     tid: 'tid',
     tradable: true,
-    gekkoBroker: 0.6
+    gekkoBroker: 0.6,
+    limitedCancelConfirmation: true
   };
 };
 

plugins/trader/trader.js

@@ -246,7 +246,7 @@ Trader.prototype.createOrder = function(side, amount, advice, id) {
 
   this.order = this.broker.createOrder(type, side, amount);
 
-  this.order.on('filled', f => log.info('[ORDER] partial', side, ' fill, total filled:', f));
+  this.order.on('fill', f => log.info('[ORDER] partial', side, 'fill, total filled:', f));
   this.order.on('statusChange', s => log.debug('[ORDER] statusChange:', s));
 
   this.order.on('error', e => {