diff --git a/core/tools/dataStitcher.js b/core/tools/dataStitcher.js index fb3995ade..1431cf1d7 100644 --- a/core/tools/dataStitcher.js +++ b/core/tools/dataStitcher.js @@ -18,6 +18,7 @@ Stitcher.prototype.ago = function(ts) { } Stitcher.prototype.verifyExchange = function() { + require(dirs.gekko + 'exchange/dependencyCheck'); const exchangeChecker = require(dirs.gekko + 'exchange/exchangeChecker'); const slug = config.watch.exchange.toLowerCase(); let exchange; diff --git a/core/workers/pipeline/child.js b/core/workers/pipeline/child.js index d1abfbb1e..92c471df2 100644 --- a/core/workers/pipeline/child.js +++ b/core/workers/pipeline/child.js @@ -56,9 +56,11 @@ process.on('disconnect', function() { process .on('unhandledRejection', (message, p) => { + console.error('unhandledRejection', message); process.send({type: 'error', message: message}); }) .on('uncaughtException', err => { + console.error('uncaughtException', err); process.send({type: 'error', error: err}); process.exit(1); }); \ No newline at end of file diff --git a/docs/extending/add_an_exchange.md b/docs/extending/add_an_exchange.md index c66336c65..eb4194c67 100644 --- a/docs/extending/add_an_exchange.md +++ b/docs/extending/add_an_exchange.md @@ -62,21 +62,17 @@ The callback needs to have the parameters of `err` and `portfolio`. Portfolio ne The callback needs to have the parameters of `err` and `lot`. Lot needs to be an object with `amount` and `purchase` size appropriately for the exchange. In the event that the lot is too small, return 0 to both fields and this will generate a lot size warning in the portfolioManager. -Note: This function is currently optional. If not implemented `portfolioManager` will fallback to basic lot sizing mechanism it uses internally. However exchanges are not all the same in how rounding and lot sizing work, it is recommend to implement this function. +Note: This function is currently optional. If not implemented Gekko Broker will fallback to basic lot sizing mechanism it uses internally. However exchanges are not all the same in how rounding and lot sizing work, it is recommend to implement this function. ### buy this.exchange.buy(amount, price, callback); -*Note that this function is a critical function, retry handlers should abort quickly if attempts to dispatch this to the exchange API fail so we don't post out of date orders to the books.* - ### sell this.exchange.sell(amount, price, callback); -This should create a buy / sell order at the exchange for [amount] of [asset] at [price] per 1 asset. If you have set `direct` to `true` the price will be `false`. The callback needs to have the parameters `err` and `order`. The order needs to be something that can be fed back to the exchange to see whether the order has been filled or not. - -*Note that this function is a critical function, retry handlers should abort quickly if attempts to dispatch this to the exchange API fail so we don't post out of date orders to the books.* +This should create a buy / sell order at the exchange for [amount] of [asset] at [price] per 1 asset. The callback needs to have the parameters `err` and `order`. The order needs to be some id that can be passed back to `checkOrder`, `getOrder` and `cancelOrder`. ### getOrder @@ -98,7 +94,7 @@ The order will be something that the manager previously received via the `sell` this.exchange.cancelOrder(order, callback); -The order will be something that the manager previously received via the `sell` or `buy` methods. The callback should have the parameters `err` and `filled`, `filled` last one should be true if the order was filled before it could be cancelled. +The order will be something that the manager previously received via the `sell` or `buy` methods. The callback should have the parameters `err`, `filled` and optionally a `fill` object, `filled` should be true if the order was fully filled before it could be cancelled. If the exchange provided how much was filled before the cancel this should be passed in the `fill object` as a `amountFilled` property. If the exchange provided how much of the original amount is still remaining, pass this as the `remaining` property instead. ### roundPrice @@ -157,6 +153,7 @@ Each exchange *must* provide a `getCapabilities()` static method that returns an - `requires`: if gekko supports automatic trading, this is an array of required api credentials gekko needs to pass into the constructor. - `forceReorderDelay`: if after canceling an order a new one can't be created straight away since the balance is not updated fast enough, set this to true (only required for exchanges where Gekko can trade). - `gekkoBroker`: set this to "0.6.2" for now, it indicates the version of Gekko Broker this wrapper is compatible with. +- `limitedCancelConfirmation` set to true if (in any case) the wrapper does NOT include partial fill information in the response (which is passed as the cancel callback) Below is a real-case example how `bistamp` exchange provides its `getCapabilities()` method: @@ -177,7 +174,8 @@ Trader.getCapabilities = function () { requires: ['key', 'secret', 'username'], fetchTimespan: 60, tid: 'tid', - gekkoBroker: '0.6.2' + gekkoBroker: '0.6.2', + limitedCancelConfirmation: false }; } ``` diff --git a/exchange/dependencyCheck.js b/exchange/dependencyCheck.js new file mode 100644 index 000000000..1cdcbc911 --- /dev/null +++ b/exchange/dependencyCheck.js @@ -0,0 +1,23 @@ +const deps = require('./package.json').dependencies; + +const missing = []; + +Object.keys(deps).forEach(dep => { + try { + require(dep); + } catch(e) { + if(e.code === 'MODULE_NOT_FOUND') { + missing.push(dep); + } + } +}); + +if(missing.length) { + console.error( + '\nThe following Gekko Broker dependencies are not installed: [', + missing.join(', '), + '].\n\nYou need to install them first, read here how:', + 'https://gekko.wizb.it/docs/installation/installing_gekko.html#Installing-Gekko-39-s-dependencies\n' + ); + process.exit(1); +} diff --git a/exchange/gekkoBroker.js b/exchange/gekkoBroker.js index 09706a52a..12749fe67 100644 --- a/exchange/gekkoBroker.js +++ b/exchange/gekkoBroker.js @@ -134,14 +134,17 @@ class Broker { if(!orders[type]) throw new Error('Unknown order type'); - const order = new orders[type](this.api); + const order = new orders[type]({ + api: this.api, + marketConfig: this.marketConfig, + capabilities: this.capabilities + }); // todo: figure out a smarter generic way this.syncPrivateData(() => { order.setData({ balances: this.portfolio.balances, ticker: this.ticker, - market: this.marketConfig }); order.create(side, amount, parameters); diff --git a/exchange/orders/order.js b/exchange/orders/order.js index 084ea13a6..4a6d5ea2f 100644 --- a/exchange/orders/order.js +++ b/exchange/orders/order.js @@ -26,7 +26,7 @@ class BaseOrder extends EventEmitter { submit({side, amount, price, alreadyFilled}) { const check = isValidOrder({ - market: this.data.market, + market: this.market, api: this.api, amount, price diff --git a/exchange/orders/sticky.js b/exchange/orders/sticky.js index b1ebd44b9..c41074c46 100644 --- a/exchange/orders/sticky.js +++ b/exchange/orders/sticky.js @@ -21,9 +21,12 @@ const BaseOrder = require('./order'); const states = require('./states'); class StickyOrder extends BaseOrder { - constructor(api) { + constructor({api, marketConfig, capabilities}) { super(api); + this.market = marketConfig; + this.capabilities = capabilities; + // global async lock this.sticking = false; @@ -126,7 +129,56 @@ class StickyOrder extends BaseOrder { }); } + // check if the last order was partially filled + // on an exchange that does not pass fill data on cancel + // see https://github.com/askmike/gekko/pull/2450 + handleInsufficientFundsError(err) { + if( + !err || + err.type !== 'insufficientFunds' || + !this.capabilities.limitedCancelConfirmation || + !this.id + ) { + return false; + } + + const id = this.id; + + setTimeout( + () => { + this.api.getOrder(id, (innerError, res) => { + if(this.handleError(innerError)) { + return; + } + + const amount = res.amount; + + if(this.orders[id].filled === amount) { + // handle original error + return this.handleError(err); + } + + this.orders[id].filled = amount; + this.emit('fill', this.calculateFilled()); + if(this.calculateFilled() >= this.amount) { + return this.filled(this.price); + } + + setTimeout(this.createOrder, this.checkInterval); + }); + }, + this.checkInterval + ); + + return true; + } + handleCreate(err, id) { + + if(this.handleInsufficientFundsError(err)) { + return; + } + if(this.handleError(err)) { return; } @@ -394,7 +446,7 @@ class StickyOrder extends BaseOrder { this.amount = this.roundAmount(amount - this.calculateFilled()); - if(this.amount < this.data.market.minimalOrder.amount) { + if(this.amount < this.market.minimalOrder.amount) { if(this.calculateFilled()) { // we already filled enough of the order! this.filled(); @@ -473,10 +525,12 @@ class StickyOrder extends BaseOrder { return next(); } - setTimeout(() => this.api.getOrder(id, next), this.timeout); + setTimeout(() => this.api.getOrder(id, next), this.checkInterval); }); async.series(checkOrders, (err, trades) => { + // note this is a standalone function after the order is + // completed, as such we do not use the handleError flow. if(err) { return next(err); } diff --git a/exchange/wrappers/binance.js b/exchange/wrappers/binance.js index d5d1520a3..07fbfa04c 100644 --- a/exchange/wrappers/binance.js +++ b/exchange/wrappers/binance.js @@ -64,6 +64,8 @@ const Trader = function(config) { this.fee = 0.1; // Set the proper fee asap. this.getFee(_.noop); + + this.oldOrder = false; } }; @@ -78,7 +80,9 @@ const recoverableErrors = [ 'Response code 5', 'Response code 403', 'ETIMEDOUT', - 'EHOSTUNREACH' + 'EHOSTUNREACH', + // getaddrinfo EAI_AGAIN api.binance.com api.binance.com:443 + 'EAI_AGAIN' ]; const includes = (str, list) => { @@ -115,6 +119,10 @@ Trader.prototype.handleResponse = function(funcName, callback) { return callback(false, {filled: true}); } + if(funcName === 'addOrder' && error.message.includes('Account has insufficient balance')) { + error.type = 'insufficientFunds'; + } + return callback(error); } @@ -299,7 +307,6 @@ Trader.prototype.isValidPrice = function(price) { } Trader.prototype.isValidLot = function(price, amount) { - console.log('isValidLot', this.market.minimalOrder.order, amount * price >= this.market.minimalOrder.order) return amount * price >= this.market.minimalOrder.order; } @@ -458,7 +465,12 @@ Trader.prototype.checkOrder = function(order, callback) { Trader.prototype.cancelOrder = function(order, callback) { const cancel = (err, data) => { + + this.oldOrder = order; + if(err) { + if(err.message.contains('')) + return callback(err); } @@ -490,7 +502,8 @@ Trader.getCapabilities = function() { providesFullHistory: true, tid: 'tid', tradable: true, - gekkoBroker: 0.6 + gekkoBroker: 0.6, + limitedCancelConfirmation: true }; }; diff --git a/exchange/wrappers/coinfalcon.js b/exchange/wrappers/coinfalcon.js index e51df65d8..df49ce1db 100644 --- a/exchange/wrappers/coinfalcon.js +++ b/exchange/wrappers/coinfalcon.js @@ -5,7 +5,10 @@ const marketData = require('./coinfalcon-markets.json'); const CoinFalcon = require('coinfalcon'); var Trader = function(config) { - _.bindAll(this); + _.bindAll(this, [ + 'roundAmount', + 'roundPrice' + ]); if (_.isObject(config)) { this.key = config.key; @@ -37,6 +40,7 @@ const recoverableErrors = [ 'SOCKETTIMEDOUT', 'TIMEDOUT', 'CONNRESET', + 'ECONNRESET', 'CONNREFUSED', 'NOTFOUND', '429', @@ -51,7 +55,9 @@ const recoverableErrors = [ 'is invalid, current timestamp is', 'EHOSTUNREACH', // https://github.com/askmike/gekko/issues/2407 - 'We are fixing a few issues, be back shortly.' + 'We are fixing a few issues, be back shortly.', + 'Client network socket disconnected before secure TLS connection was established', + 'socket hang up' ]; Trader.prototype.processResponse = function(method, args, next) { @@ -62,7 +68,7 @@ Trader.prototype.processResponse = function(method, args, next) { if(includes(err.message, recoverableErrors)) return this.retry(method, args); - console.log(new Date, '[cf] big error!', err); + console.log(new Date, '[cf] big error!', err.message); return next(err); } @@ -166,18 +172,25 @@ const round = function(number, precision) { return roundedTempNumber / factor; }; +// ticksize 0.01 will yield: 2 +Trader.prototype.getPrecision = function(tickSize) { + if (!isFinite(tickSize)) { + return 0; + } + var e = 1; + var p = 0; + while (Math.round(tickSize * e) / e !== tickSize) { + e *= 10; p++; + } + return p; +}; + Trader.prototype.roundAmount = function(amount) { - return round(amount, 8); + return round(amount, this.getPrecision(this.market.minimalOrder.amount)); } Trader.prototype.roundPrice = function(price) { - let rounding; - - if(this.pair.includes('EUR')) - rounding = 2; - else - rounding = 5; - return round(price, rounding); + return round(price, this.getPrecision(this.market.minimalOrder.price)); } Trader.prototype.outbidPrice = function(price, isUp) { diff --git a/exchange/wrappers/kraken.js b/exchange/wrappers/kraken.js index 7f26583c3..53d65900f 100644 --- a/exchange/wrappers/kraken.js +++ b/exchange/wrappers/kraken.js @@ -41,7 +41,8 @@ const recoverableErrors = [ 'Request timed out', 'Response code 5', 'Empty response', - 'API:Invalid nonce' + 'API:Invalid nonce', + 'General:Temporary lockout' ]; const includes = (str, list) => { diff --git a/exchange/wrappers/poloniex.js b/exchange/wrappers/poloniex.js index 2f1d70fae..8a6f89c2d 100644 --- a/exchange/wrappers/poloniex.js +++ b/exchange/wrappers/poloniex.js @@ -37,6 +37,7 @@ const recoverableErrors = [ '503', '500', '502', + 'socket hang up', 'Empty response', 'Please try again in a few minutes.', 'Nonce must be greater than', diff --git a/package-lock.json b/package-lock.json index 4c6a1f9a5..c59fe1f87 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "gekko", - "version": "0.6.4", + "version": "0.6.5", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/plugins/performanceAnalyzer/logger.js b/plugins/performanceAnalyzer/logger.js index e087c39c5..5db6e10da 100644 --- a/plugins/performanceAnalyzer/logger.js +++ b/plugins/performanceAnalyzer/logger.js @@ -29,10 +29,10 @@ Logger.prototype.logReport = function(trade, report) { var start = this.round(report.startBalance); var current = this.round(report.balance); - log.info(`(PROFIT REPORT) original balance:\t ${start} ${this.currency}`); - log.info(`(PROFIT REPORT) current balance:\t ${current} ${this.currency}`); + log.info(`(PROFIT REPORT) original balance:\t\t ${start} ${this.currency}`); + log.info(`(PROFIT REPORT) current balance:\t\t ${current} ${this.currency}`); log.info( - `(PROFIT REPORT) profit:\t\t ${this.round(report.profit)} ${this.currency}`, + `(PROFIT REPORT) profit:\t\t\t\t ${this.round(report.profit)} ${this.currency}`, `(${this.round(report.relativeProfit)}%)` ); } diff --git a/plugins/trader/trader.js b/plugins/trader/trader.js index cc1139c48..7d1b6ba67 100644 --- a/plugins/trader/trader.js +++ b/plugins/trader/trader.js @@ -7,6 +7,8 @@ const moment = require('moment'); const log = require(dirs.core + 'log'); const Broker = require(dirs.gekko + '/exchange/gekkoBroker'); +require(dirs.gekko + '/exchange/dependencyCheck'); + const Trader = function(next) { _.bindAll(this); @@ -246,7 +248,7 @@ Trader.prototype.createOrder = function(side, amount, advice, id) { this.order = this.broker.createOrder(type, side, amount); - this.order.on('filled', f => log.info('[ORDER] partial', side, ' fill, total filled:', f)); + this.order.on('fill', f => log.info('[ORDER] partial', side, 'fill, total filled:', f)); this.order.on('statusChange', s => log.debug('[ORDER] statusChange:', s)); this.order.on('error', e => { diff --git a/web/server.js b/web/server.js index 5db10152b..140b3e193 100644 --- a/web/server.js +++ b/web/server.js @@ -20,24 +20,50 @@ const cache = require('./state/cache'); const nodeCommand = _.last(process.argv[1].split('/')); const isDevServer = nodeCommand === 'server' || nodeCommand === 'server.js'; +wss.on('connection', ws => { + ws.isAlive = true; + ws.on('pong', () => { + ws.isAlive = true; + }); + ws.ping(_.noop); + ws.on('error', e => { + console.error(new Date, '[WS] connection error:', e); + }); +}); + + +setInterval(() => { + wss.clients.forEach(ws => { + if(!ws.isAlive) { + console.log(new Date, '[WS] stale websocket client, terminiating..'); + return ws.terminate(); + } + + ws.isAlive = false; + ws.ping(_.noop); + }); +}, 10 * 1000); + // broadcast function const broadcast = data => { - if(_.isEmpty(data)) + if(_.isEmpty(data)) { return; + } + + const payload = JSON.stringify(data); - _.each( - wss.clients, - client => { - try { - client.send(JSON.stringify(data)); - } catch(e) { - log.warn('unable to send data to client'); + wss.clients.forEach(ws => { + ws.send(payload, err => { + if(err) { + console.log(new Date, '[WS] unable to send data to client:', err); } - } + }); + } ); } cache.set('broadcast', broadcast); + const ListManager = require('./state/listManager'); const GekkoManager = require('./state/gekkoManager');