diff --git a/indexer/packages/redis/__tests__/caches/canceled-orders-cache.test.ts b/indexer/packages/redis/__tests__/caches/canceled-orders-cache.test.ts index 41f1a7b6972..e0b1b73ba8a 100644 --- a/indexer/packages/redis/__tests__/caches/canceled-orders-cache.test.ts +++ b/indexer/packages/redis/__tests__/caches/canceled-orders-cache.test.ts @@ -4,8 +4,11 @@ import { CANCELED_ORDER_WINDOW_SIZE, isOrderCanceled, addCanceledOrderId, - removeOrderFromCache, + removeOrderFromCaches, + getOrderCanceledStatus, + addBestEffortCanceledOrderId, } from '../../src/caches/canceled-orders-cache'; +import { CanceledOrderStatus } from '../../src'; describe('cancelledOrdersCache', () => { const openOrderId1: string = 'order1'; @@ -40,13 +43,13 @@ describe('cancelledOrdersCache', () => { expect(isCanceled1).toEqual(true); expect(isCanceled2).toEqual(true); - let numRemoved: number = await removeOrderFromCache(openOrderId1, client); - expect(numRemoved).toEqual(1); + await removeOrderFromCaches(openOrderId1, client); isCanceled1 = await isOrderCanceled(openOrderId1, client); expect(isCanceled1).toEqual(false); - numRemoved = await removeOrderFromCache(openOrderId3, client); - expect(numRemoved).toEqual(0); + await removeOrderFromCaches(openOrderId3, client); + const isCanceled3: boolean = await isOrderCanceled(openOrderId1, client); + expect(isCanceled3).toEqual(false); }); it('removes cancelled orders outside of window size', async () => { @@ -61,4 +64,22 @@ describe('cancelledOrdersCache', () => { expect(isCanceled3).toEqual(true); }); + describe('getOrderCanceledStatus', () => { + it('correctly returns CANCELED', async () => { + await addCanceledOrderId(openOrderId1, 10, client); + const status: CanceledOrderStatus = await getOrderCanceledStatus(openOrderId1, client); + expect(status).toEqual(CanceledOrderStatus.CANCELED); + }); + + it('correctly returns BEST_EFFORT_CANCELED', async () => { + await addBestEffortCanceledOrderId(openOrderId1, 10, client); + const status: CanceledOrderStatus = await getOrderCanceledStatus(openOrderId1, client); + expect(status).toEqual(CanceledOrderStatus.BEST_EFFORT_CANCELED); + }); + + it('correctly returns NOT_CANCELED', async () => { + const status: CanceledOrderStatus = await getOrderCanceledStatus(openOrderId1, client); + expect(status).toEqual(CanceledOrderStatus.NOT_CANCELED); + }); + }); }); diff --git a/indexer/packages/redis/src/caches/canceled-orders-cache.ts b/indexer/packages/redis/src/caches/canceled-orders-cache.ts index f3d19a02a1a..e2bf109336f 100644 --- a/indexer/packages/redis/src/caches/canceled-orders-cache.ts +++ b/indexer/packages/redis/src/caches/canceled-orders-cache.ts @@ -1,9 +1,11 @@ import { Callback, RedisClient } from 'redis'; import { zRemAsync, zScoreAsync } from '../helpers/redis'; +import { CanceledOrderStatus } from '../types'; import { addCanceledOrderIdScript } from './scripts'; // Cache of cancelled orders export const CANCELED_ORDERS_CACHE_KEY: string = 'v4/cancelled_orders'; +export const BEST_EFFORT_CANCELED_ORDERS_CACHE_KEY: string = 'v4/best_effort_cancelled_orders'; // 10 seconds in milliseconds export const CANCELED_ORDER_WINDOW_SIZE: number = 30 * 1000; @@ -24,16 +26,62 @@ export async function isOrderCanceled( orderId: string, client: RedisClient, ): Promise { - const score: string | null = await - zScoreAsync({ hash: CANCELED_ORDERS_CACHE_KEY, key: orderId }, client); - return score !== null; + const [ + canceledScore, + bestEffortCanceledScore, + ]: (string | null)[] = await Promise.all([ + zScoreAsync({ hash: CANCELED_ORDERS_CACHE_KEY, key: orderId }, client), + zScoreAsync({ hash: BEST_EFFORT_CANCELED_ORDERS_CACHE_KEY, key: orderId }, client), + ]); + return canceledScore !== null || bestEffortCanceledScore !== null; } -export async function removeOrderFromCache( +export async function getOrderCanceledStatus( orderId: string, client: RedisClient, +): Promise { + const [ + canceledScore, + bestEffortCanceledScore, + ]: (string | null)[] = await Promise.all([ + zScoreAsync({ hash: CANCELED_ORDERS_CACHE_KEY, key: orderId }, client), + zScoreAsync({ hash: BEST_EFFORT_CANCELED_ORDERS_CACHE_KEY, key: orderId }, client), + ]); + + if (canceledScore !== null) { + return CanceledOrderStatus.CANCELED; + } + + if (bestEffortCanceledScore !== null) { + return CanceledOrderStatus.BEST_EFFORT_CANCELED; + } + + return CanceledOrderStatus.NOT_CANCELED; +} + +export async function removeOrderFromCaches( + orderId: string, + client: RedisClient, +): Promise { + await Promise.all([ + zRemAsync({ hash: CANCELED_ORDERS_CACHE_KEY, key: orderId }, client), + zRemAsync({ hash: BEST_EFFORT_CANCELED_ORDERS_CACHE_KEY, key: orderId }, client), + ]); +} + +/** + * addCanceledOrderId adds the order id to the best effort canceled orders cache. + * + * @param orderId + * @param timestamp + * @param client + */ +export async function addBestEffortCanceledOrderId( + orderId: string, + timestamp: number, + client: RedisClient, ): Promise { - return zRemAsync({ hash: CANCELED_ORDERS_CACHE_KEY, key: orderId }, client); + return addOrderIdtoCache(orderId, timestamp, client, BEST_EFFORT_CANCELED_ORDERS_CACHE_KEY); } /** @@ -47,6 +95,22 @@ export async function addCanceledOrderId( orderId: string, timestamp: number, client: RedisClient, +): Promise { + return addOrderIdtoCache(orderId, timestamp, client, CANCELED_ORDERS_CACHE_KEY); +} + +/** + * addCanceledOrderId adds the order id to the cacheKey's cache. + * + * @param orderId + * @param timestamp + * @param client + */ +export async function addOrderIdtoCache( + orderId: string, + timestamp: number, + client: RedisClient, + cacheKey: string, ): Promise { const numKeys: number = 2; let evalAsync: ( @@ -69,7 +133,7 @@ export async function addCanceledOrderId( client.evalsha( addCanceledOrderIdScript.hash, numKeys, - CANCELED_ORDERS_CACHE_KEY, + cacheKey, CANCELED_ORDER_WINDOW_SIZE, canceledOrderId, currentTimestampMs, diff --git a/indexer/packages/redis/src/types.ts b/indexer/packages/redis/src/types.ts index 0f67c7788de..76e2339d271 100644 --- a/indexer/packages/redis/src/types.ts +++ b/indexer/packages/redis/src/types.ts @@ -67,6 +67,12 @@ export type LuaScript = { readonly hash: string; }; +export enum CanceledOrderStatus { + CANCELED = 'CANCELED', + BEST_EFFORT_CANCELED = 'BEST_EFFORT_CANCELED', + NOT_CANCELED = 'NOT_CANCELED', +} + /* ------- PNL Creation TYPES ------- */ export type PnlTickForSubaccounts = { // Stores a PnlTicksCreateObject for the most recent pnl tick for each subaccount. diff --git a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts index b22e85afda8..589a1950bf5 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts @@ -707,7 +707,7 @@ describe('OrderHandler', () => { totalFilled: totalMakerOrderFilled, price, status: isOrderCanceled - ? OrderStatus.BEST_EFFORT_CANCELED + ? OrderStatus.CANCELED : OrderStatus.OPEN, // orderSize > totalFilled so status is open clobPairId: defaultClobPairId, side: protocolTranslations.protocolOrderSideToOrderSide(makerOrderProto.side), @@ -1359,6 +1359,258 @@ describe('OrderHandler', () => { await expectNoCandles(); }); + it.each([ + [ + 'via knex', + false, + ], + [ + 'via SQL function', + true, + ], + ])('correctly sets status for short term IOC orders (%s)', async ( + _name: string, + useSqlFunction: boolean, + ) => { + config.USE_ORDER_HANDLER_SQL_FUNCTION = useSqlFunction; + const transactionIndex: number = 0; + const eventIndex: number = 0; + const makerQuantums: number = 100; + const makerSubticks: number = 1_000_000; + + const makerOrderProto: IndexerOrder = createOrder({ + subaccountId: defaultSubaccountId, + clientId: 0, + side: IndexerOrder_Side.SIDE_BUY, + quantums: makerQuantums, + subticks: makerSubticks, + goodTilOneof: { + goodTilBlock: 10, + }, + clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, + orderFlags: ORDER_FLAG_SHORT_TERM.toString(), + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, + reduceOnly: false, + clientMetadata: 0, + }); + + const takerSubticks: number = 150_000; + const takerQuantums: number = 10; + const takerOrderProto: IndexerOrder = createOrder({ + subaccountId: defaultSubaccountId2, + clientId: 0, + side: IndexerOrder_Side.SIDE_SELL, + quantums: takerQuantums, + subticks: takerSubticks, + goodTilOneof: { + goodTilBlock: 10, + }, + clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, + orderFlags: ORDER_FLAG_SHORT_TERM.toString(), + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, + reduceOnly: true, + clientMetadata: 0, + }); + + const fillAmount: number = takerQuantums; + const orderFillEvent: OrderFillEventV1 = createOrderFillEvent( + makerOrderProto, + takerOrderProto, + fillAmount, + fillAmount, + fillAmount, + ); + const kafkaMessage: KafkaMessage = createKafkaMessageFromOrderFillEvent({ + orderFillEvent, + transactionIndex, + eventIndex, + height: parseInt(defaultHeight, 10), + time: defaultTime, + txHash: defaultTxHash, + }); + + await Promise.all([ + // initial position for subaccount 1 + PerpetualPositionTable.create({ + ...defaultPerpetualPosition, + perpetualId: testConstants.defaultPerpetualMarket3.id, + }), + // initial position for subaccount 2 + PerpetualPositionTable.create({ + ...defaultPerpetualPosition, + subaccountId: testConstants.defaultSubaccountId2, + perpetualId: testConstants.defaultPerpetualMarket3.id, + }), + ]); + + await onMessage(kafkaMessage); + + const makerOrderId: string = OrderTable.orderIdToUuid(makerOrderProto.orderId!); + const takerOrderId: string = OrderTable.orderIdToUuid(takerOrderProto.orderId!); + + const [makerOrder, takerOrder]: [ + OrderFromDatabase | undefined, + OrderFromDatabase | undefined + ] = await Promise.all([ + OrderTable.findById(makerOrderId), + OrderTable.findById(takerOrderId), + ]); + + expect(makerOrder).toBeDefined(); + expect(takerOrder).toBeDefined(); + + // maker order is partially filled + expect(makerOrder!.status).toEqual(OrderStatus.CANCELED); + // taker order is fully filled + expect(takerOrder!.status).toEqual(OrderStatus.FILLED); + }); + + it.each([ + [ + 'limit', + 'via knex', + false, + IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, + ], + [ + 'limit', + 'via SQL function', + true, + IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, + ], + [ + 'post-only best effort canceled', + 'via knex', + false, + IndexerOrder_TimeInForce.TIME_IN_FORCE_POST_ONLY, + ], + [ + 'post-only best effort canceled', + 'via SQL function', + true, + IndexerOrder_TimeInForce.TIME_IN_FORCE_POST_ONLY, + ], + [ + 'post-only canceled', + 'via knex', + false, + IndexerOrder_TimeInForce.TIME_IN_FORCE_POST_ONLY, + OrderStatus.CANCELED, + ], + [ + 'post-only canceled', + 'via SQL function', + true, + IndexerOrder_TimeInForce.TIME_IN_FORCE_POST_ONLY, + OrderStatus.CANCELED, + ], + ])('correctly sets status for short term %s orders (%s)', async ( + _orderType: string, + _name: string, + useSqlFunction: boolean, + timeInForce: IndexerOrder_TimeInForce, + // either BEST_EFFORT_CANCELED or CANCELED + status: OrderStatus = OrderStatus.BEST_EFFORT_CANCELED, + ) => { + config.USE_ORDER_HANDLER_SQL_FUNCTION = useSqlFunction; + const transactionIndex: number = 0; + const eventIndex: number = 0; + const makerQuantums: number = 100; + const makerSubticks: number = 1_000_000; + + const makerOrderProto: IndexerOrder = createOrder({ + subaccountId: defaultSubaccountId, + clientId: 0, + side: IndexerOrder_Side.SIDE_BUY, + quantums: makerQuantums, + subticks: makerSubticks, + goodTilOneof: { + goodTilBlock: 10, + }, + clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, + orderFlags: ORDER_FLAG_SHORT_TERM.toString(), + timeInForce, + reduceOnly: false, + clientMetadata: 0, + }); + + const takerSubticks: number = 150_000; + const takerQuantums: number = 100; + const takerOrderProto: IndexerOrder = createOrder({ + subaccountId: defaultSubaccountId2, + clientId: 0, + side: IndexerOrder_Side.SIDE_SELL, + quantums: takerQuantums, + subticks: takerSubticks, + goodTilOneof: { + goodTilBlock: 10, + }, + clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, + orderFlags: ORDER_FLAG_SHORT_TERM.toString(), + timeInForce, + reduceOnly: true, + clientMetadata: 0, + }); + + const makerOrderId: string = OrderTable.orderIdToUuid(makerOrderProto.orderId!); + if (status === OrderStatus.BEST_EFFORT_CANCELED) { + await CanceledOrdersCache.addBestEffortCanceledOrderId(makerOrderId, Date.now(), redisClient); + } else { // Status is only over CANCELED or BEST_EFFORT_CANCELED + await CanceledOrdersCache.addCanceledOrderId(makerOrderId, Date.now(), redisClient); + } + + const fillAmount: number = 10; + const orderFillEvent: OrderFillEventV1 = createOrderFillEvent( + makerOrderProto, + takerOrderProto, + fillAmount, + fillAmount, + fillAmount, + ); + const kafkaMessage: KafkaMessage = createKafkaMessageFromOrderFillEvent({ + orderFillEvent, + transactionIndex, + eventIndex, + height: parseInt(defaultHeight, 10), + time: defaultTime, + txHash: defaultTxHash, + }); + + await Promise.all([ + // initial position for subaccount 1 + PerpetualPositionTable.create({ + ...defaultPerpetualPosition, + perpetualId: testConstants.defaultPerpetualMarket3.id, + }), + // initial position for subaccount 2 + PerpetualPositionTable.create({ + ...defaultPerpetualPosition, + subaccountId: testConstants.defaultSubaccountId2, + perpetualId: testConstants.defaultPerpetualMarket3.id, + }), + ]); + + await onMessage(kafkaMessage); + + const takerOrderId: string = OrderTable.orderIdToUuid(takerOrderProto.orderId!); + + const [makerOrder, takerOrder]: [ + OrderFromDatabase | undefined, + OrderFromDatabase | undefined + ] = await Promise.all([ + OrderTable.findById(makerOrderId), + OrderTable.findById(takerOrderId), + ]); + + expect(makerOrder).toBeDefined(); + expect(takerOrder).toBeDefined(); + + // maker order is partially filled, and in CanceledOrdersCache + expect(makerOrder!.status).toEqual(status); + // taker order is partially filled, and not in CanceledOrdersCache + expect(takerOrder!.status).toEqual(OrderStatus.OPEN); + }); + async function expectDefaultOrderAndFillSubaccountKafkaMessages( producerSendMock: jest.SpyInstance, eventId: Buffer, diff --git a/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts b/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts index 1d42fe4da05..52814bc5bf1 100644 --- a/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts +++ b/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts @@ -22,11 +22,13 @@ import { SubaccountMessageContents, SubaccountTable, TendermintEventTable, + TimeInForce, TradeMessageContents, UpdatedPerpetualPositionSubaccountKafkaObject, USDC_ASSET_ID, } from '@dydxprotocol-indexer/postgres'; -import { getOrderIdHash } from '@dydxprotocol-indexer/v4-proto-parser'; +import { CanceledOrderStatus } from '@dydxprotocol-indexer/redis'; +import { getOrderIdHash, ORDER_FLAG_LONG_TERM } from '@dydxprotocol-indexer/v4-proto-parser'; import { IndexerOrder, IndexerOrder_Side, @@ -289,11 +291,16 @@ export abstract class AbstractOrderFillHandler extends Handler { return updatedPerpetualPosition; } + /** + * Upsert the an order based on the event processed by the handler + * @param canceledOrderStatus - Status of the order in the CanceledOrderCache, always + * NOT_CANCELED for liquidation orders + */ protected upsertOrderFromEvent( perpetualMarket: PerpetualMarketFromDatabase, order: IndexerOrder, totalFilledFromProto: Long, - isCanceled: boolean, + canceledOrderStatus: CanceledOrderStatus, ): Promise { const size: string = getSize(order, perpetualMarket); const price: string = getPrice(order, perpetualMarket); @@ -301,6 +308,14 @@ export abstract class AbstractOrderFillHandler extends Handler { totalFilledFromProto.toString(10), perpetualMarket.atomicResolution, ); + const timeInForce: TimeInForce = protocolTranslations.protocolOrderTIFToTIF(order.timeInForce); + const status: OrderStatus = this.getOrderStatus( + canceledOrderStatus, + size, + totalFilled, + order.orderId!.orderFlags, + timeInForce, + ); const orderToCreate: OrderCreateObject = { subaccountId: SubaccountTable.subaccountIdToUuid(order.orderId!.subaccountId!), @@ -311,8 +326,8 @@ export abstract class AbstractOrderFillHandler extends Handler { totalFilled, price, type: OrderType.LIMIT, // TODO: Add additional order types once we support - status: this.getOrderStatus(isCanceled, size, totalFilled), - timeInForce: protocolTranslations.protocolOrderTIFToTIF(order.timeInForce), + status, + timeInForce, reduceOnly: order.reduceOnly, orderFlags: order.orderId!.orderFlags.toString(), goodTilBlock: protocolTranslations.getGoodTilBlock(order)?.toString(), @@ -325,16 +340,44 @@ export abstract class AbstractOrderFillHandler extends Handler { return OrderTable.upsert(orderToCreate, { txId: this.txId }); } + /** + * The obvious case is if totalFilled >= size, then the order status should always be `FILLED`. + * The difficult case is if totalFilled < size after a fill, then we need to keep the following + * cases in mind: + * 1. Stateful Orders - All cancelations are on-chain events, so the will be `OPEN`. The + * CanceledOrdersCache does not store any stateful orders and we never send + * BEST_EFFORT_CANCELED notifications for stateful orders. + * 2. Short-term FOK - FOK orders can never be `OPEN`, since they don't rest on the orderbook, so + * totalFilled cannot be < size. By the end of the block, the order will be filled, so we mark + * it as `FILLED`. + * 3. Short-term IOC - Protocol guarantees that an IOC order will only ever be filled in a single + * block, so status should be `CANCELED`. + * 4. Short-term Limit & Post-only - If the order is in the CanceledOrdersCache, then it should be + * set to the corresponding CanceledOrderStatus, otherwise `OPEN`. + * @param isCanceled - if the order is in the CanceledOrderCache, always false for liquidiation + * orders + */ protected getOrderStatus( - isCanceled: boolean, + canceledOrderStatus: CanceledOrderStatus, size: string, totalFilled: string, + orderFlags: number, + timeInForce: TimeInForce, ): OrderStatus { - if (isCanceled) { - return OrderStatus.BEST_EFFORT_CANCELED; - } - if (Big(size).lte(totalFilled)) { + if (Big(totalFilled).gte(size)) { return OrderStatus.FILLED; + } else if (orderFlags === ORDER_FLAG_LONG_TERM) { // 1. Stateful Order + return OrderStatus.OPEN; + } else if (timeInForce === TimeInForce.FOK) { // 2. Short-term FOK + return OrderStatus.FILLED; + } else if (timeInForce === TimeInForce.IOC) { // 3. Short-term IOC + return OrderStatus.CANCELED; + } + // 4. Short-term Limit & Post-only + if (canceledOrderStatus === CanceledOrderStatus.BEST_EFFORT_CANCELED) { + return OrderStatus.BEST_EFFORT_CANCELED; + } else if (canceledOrderStatus === CanceledOrderStatus.CANCELED) { + return OrderStatus.CANCELED; } return OrderStatus.OPEN; } diff --git a/indexer/services/ender/src/handlers/order-fills/liquidation-handler.ts b/indexer/services/ender/src/handlers/order-fills/liquidation-handler.ts index cf4ea31a8b5..4e5ca2d4250 100644 --- a/indexer/services/ender/src/handlers/order-fills/liquidation-handler.ts +++ b/indexer/services/ender/src/handlers/order-fills/liquidation-handler.ts @@ -16,6 +16,7 @@ import { USDC_ASSET_ID, OrderStatus, FillType, } from '@dydxprotocol-indexer/postgres'; +import { CanceledOrderStatus } from '@dydxprotocol-indexer/redis'; import { isStatefulOrder } from '@dydxprotocol-indexer/v4-proto-parser'; import { LiquidationOrderV1, IndexerOrderId, OrderFillEventV1, @@ -209,7 +210,7 @@ export class LiquidationHandler extends AbstractOrderFillHandler { @@ -178,8 +181,11 @@ export class OrderHandler extends AbstractOrderFillHandler= size, then the order status should always be `FILLED`. + * The difficult case is if totalFilled < size after a fill, then we need to keep the following + * cases in mind: + * 1. Stateful Orders - All cancelations are on-chain events, so the will be `OPEN`. The + * CanceledOrdersCache does not store any stateful orders and we never send + * BEST_EFFORT_CANCELED notifications for stateful orders. + * 2. Short-term FOK - FOK orders can never be `OPEN`, since they don't rest on the orderbook, so + * totalFilled cannot be < size. By the end of the block, the order will be filled, so we mark + * it as `FILLED`. + * 3. Short-term IOC - Protocol guarantees that an IOC order will only ever be filled in a single + * block, so status should be `CANCELED`. + * 4. Short-term Limit & Post-only - If the order is in the CanceledOrdersCache, then it should be + * set to the corresponding CanceledOrderStatus, otherwise `OPEN`. + * @param isCanceled - if the order is in the CanceledOrderCache, always false for liquidiation + * orders + */ +CREATE OR REPLACE FUNCTION dydx_get_order_status(total_filled numeric, size numeric, order_canceled_status text, order_flags bigint, time_in_force text) RETURNS text AS $$ DECLARE order_status text; BEGIN - IF is_cancelled = true THEN - order_status = 'BEST_EFFORT_CANCELED'; - ELSIF total_filled >= size THEN - order_status = 'FILLED'; + IF total_filled >= size THEN + RETURN 'FILLED'; + /** Order flag of 64 is a stateful term order */ + ELSIF order_flags = 64 THEN /** 1. Stateful Order */ + RETURN 'OPEN'; + ELSIF time_in_force = 'FOK' THEN /** 2. Short-term FOK */ + RETURN 'FILLED'; + ELSIF time_in_force = 'IOC' THEN /** 3. Short-term IOC */ + RETURN 'CANCELED'; + ELSIF order_canceled_status = 'BEST_EFFORT_CANCELED' THEN /** 4. Short-term Limit & Postonly */ + RETURN 'BEST_EFFORT_CANCELED'; + ELSIF order_canceled_status = 'CANCELED' THEN + RETURN 'CANCELED'; ELSE order_status = 'OPEN'; END IF; diff --git a/indexer/services/ender/src/scripts/dydx_liquidation_fill_handler_per_order.sql b/indexer/services/ender/src/scripts/dydx_liquidation_fill_handler_per_order.sql index 2eb5e52957d..745edda0b00 100644 --- a/indexer/services/ender/src/scripts/dydx_liquidation_fill_handler_per_order.sql +++ b/indexer/services/ender/src/scripts/dydx_liquidation_fill_handler_per_order.sql @@ -119,7 +119,7 @@ BEGIN IF FOUND THEN order_record."totalFilled" = total_filled; - order_record."status" = dydx_get_order_status(total_filled, order_record.size, false, order_record."orderFlags", order_record."timeInForce"); + order_record."status" = dydx_get_order_status(total_filled, order_record.size, 'NOT_CANCELED', order_record."orderFlags", order_record."timeInForce"); UPDATE orders SET @@ -145,7 +145,7 @@ BEGIN order_record."type" = 'LIMIT'; order_record."totalFilled" = fill_amount; - order_record."status" = dydx_get_order_status(fill_amount, order_size, false, order_record."orderFlags", order_record."timeInForce"); + order_record."status" = dydx_get_order_status(fill_amount, order_size, 'NOT_CANCELED', order_record."orderFlags", order_record."timeInForce"); order_record."createdAtHeight" = block_height; INSERT INTO orders ("id", "subaccountId", "clientId", "clobPairId", "side", "size", "totalFilled", "price", "type", diff --git a/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql b/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql index 7c5e0eb1e04..ddb45e55d4c 100644 --- a/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql +++ b/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql @@ -12,7 +12,7 @@ - fill_liquidity: The liquidity for the fill record. - fill_type: The type for the fill record. - usdc_asset_id: The USDC asset id. - - is_cancelled: Whether the order is cancelled. + - order_canceled_status: Status of order cancelation Returns: JSON object containing fields: - order: The updated order in order-model format (https://github.com/dydxprotocol/indexer/blob/cc70982/packages/postgres/src/models/order-model.ts). - fill: The updated fill in fill-model format (https://github.com/dydxprotocol/indexer/blob/cc70982/packages/postgres/src/models/fill-model.ts). @@ -21,7 +21,7 @@ */ CREATE OR REPLACE FUNCTION dydx_order_fill_handler_per_order( field text, block_height int, block_time timestamp, event_data jsonb, event_index int, transaction_index int, - transaction_hash text, fill_liquidity text, fill_type text, usdc_asset_id text, is_cancelled boolean) RETURNS jsonb AS $$ + transaction_hash text, fill_liquidity text, fill_type text, usdc_asset_id text, order_canceled_status text) RETURNS jsonb AS $$ DECLARE order_ jsonb; maker_order jsonb; @@ -105,7 +105,7 @@ BEGIN IF FOUND THEN order_record."totalFilled" = total_filled; - order_record."status" = dydx_get_order_status(total_filled, order_record.size, is_cancelled, order_record."orderFlags", order_record."timeInForce"); + order_record."status" = dydx_get_order_status(total_filled, order_record.size, order_canceled_status, order_record."orderFlags", order_record."timeInForce"); UPDATE orders SET @@ -131,7 +131,7 @@ BEGIN order_record."type" = 'LIMIT'; /* TODO: Add additional order types once we support */ order_record."totalFilled" = fill_amount; - order_record."status" = dydx_get_order_status(fill_amount, order_size, is_cancelled, order_record."orderFlags", order_record."timeInForce"); + order_record."status" = dydx_get_order_status(fill_amount, order_size, order_canceled_status, order_record."orderFlags", order_record."timeInForce"); order_record."createdAtHeight" = block_height; INSERT INTO orders ("id", "subaccountId", "clientId", "clobPairId", "side", "size", "totalFilled", "price", "type", diff --git a/indexer/services/vulcan/__tests__/handlers/order-place-handler.test.ts b/indexer/services/vulcan/__tests__/handlers/order-place-handler.test.ts index 8c187cac752..8ffc6337cc1 100644 --- a/indexer/services/vulcan/__tests__/handlers/order-place-handler.test.ts +++ b/indexer/services/vulcan/__tests__/handlers/order-place-handler.test.ts @@ -41,6 +41,7 @@ import { CanceledOrdersCache, updateOrder, StatefulOrderUpdatesCache, + CanceledOrderStatus, } from '@dydxprotocol-indexer/redis'; import { @@ -58,7 +59,7 @@ import Long from 'long'; import { convertToRedisOrder, getTriggerPrice } from '../../src/handlers/helpers'; import { redisClient, redisClient as client } from '../../src/helpers/redis/redis-controller'; import { onMessage } from '../../src/lib/on-message'; -import { expectCanceledOrdersCacheEmpty, expectOpenOrderIds, handleInitialOrderPlace } from '../helpers/helpers'; +import { expectCanceledOrderStatus, expectOpenOrderIds, handleInitialOrderPlace } from '../helpers/helpers'; import { expectOffchainUpdateMessage, expectWebsocketOrderbookMessage, expectWebsocketSubaccountMessage } from '../helpers/websocket-helpers'; import { OrderbookSide } from '../../src/lib/types'; import { getOrderIdHash } from '@dydxprotocol-indexer/v4-proto-parser'; @@ -178,7 +179,7 @@ describe('order-place-handler', () => { ]); jest.spyOn(stats, 'timing'); jest.spyOn(OrderbookLevelsCache, 'updatePriceLevel'); - jest.spyOn(CanceledOrdersCache, 'removeOrderFromCache'); + jest.spyOn(CanceledOrdersCache, 'removeOrderFromCaches'); jest.spyOn(stats, 'increment'); jest.spyOn(redisPackage, 'placeOrder'); jest.spyOn(logger, 'error'); @@ -381,9 +382,9 @@ describe('order-place-handler', () => { ); expect(OrderbookLevelsCache.updatePriceLevel).not.toHaveBeenCalled(); if (hasCanceledOrderId) { - expect(CanceledOrdersCache.removeOrderFromCache).toHaveBeenCalled(); + expect(CanceledOrdersCache.removeOrderFromCaches).toHaveBeenCalled(); } - await expectCanceledOrdersCacheEmpty(expectedOrderUuid); + await expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.NOT_CANCELED); expect(logger.error).not.toHaveBeenCalled(); expectWebsocketMessagesSent( diff --git a/indexer/services/vulcan/__tests__/handlers/order-remove-handler.test.ts b/indexer/services/vulcan/__tests__/handlers/order-remove-handler.test.ts index 095e0901e1c..fffc3a19f0c 100644 --- a/indexer/services/vulcan/__tests__/handlers/order-remove-handler.test.ts +++ b/indexer/services/vulcan/__tests__/handlers/order-remove-handler.test.ts @@ -39,6 +39,7 @@ import { redisTestConstants, SubaccountOrderIdsCache, updateOrder, + CanceledOrderStatus, } from '@dydxprotocol-indexer/redis'; import { OffChainUpdateV1, @@ -58,8 +59,7 @@ import { OrderRemoveHandler } from '../../src/handlers/order-remove-handler'; import { OrderbookSide } from '../../src/lib/types'; import { redisClient } from '../../src/helpers/redis/redis-controller'; import { - expectCanceledOrdersCacheEmpty, - expectCanceledOrdersCacheFound, + expectCanceledOrderStatus, expectOpenOrderIds, expectOrderbookLevelCache, handleOrderUpdate, @@ -308,7 +308,7 @@ describe('OrderRemoveHandler', () => { expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), // Check order is removed from open orders cache expectOpenOrderIds(testConstants.defaultPerpetualMarket.clobPairId, []), - expectCanceledOrdersCacheFound(expectedOrderUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.CANCELED), ]); // Subaccounts message is sent first followed by orderbooks message @@ -442,6 +442,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.BEST_EFFORT_CANCELED), ]); // Subaccounts message is sent first followed by orderbooks message @@ -577,6 +578,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.CANCELED), ]); // Subaccounts message is sent first followed by orderbooks message @@ -711,6 +713,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.CANCELED), ]); // Subaccounts message is sent first followed by orderbooks message @@ -846,6 +849,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.BEST_EFFORT_CANCELED), ]); // no orderbook message because no change in orderbook levels @@ -928,6 +932,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.NOT_CANCELED), ]); // no orderbook message because no change in orderbook levels @@ -1107,7 +1112,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), - expectCanceledOrdersCacheEmpty(expectedOrderUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.NOT_CANCELED), ]); // Subaccounts message is sent first followed by orderbooks message @@ -1226,6 +1231,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.NOT_CANCELED), ]); // Subaccounts message is sent first followed by orderbooks message @@ -1360,6 +1366,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.NOT_CANCELED), ]); // Subaccounts message is sent first followed by orderbooks message @@ -1485,6 +1492,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.CANCELED), ]); // Subaccounts message is sent first followed by orderbooks message @@ -1602,6 +1610,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheEmpty(expectedOrderUuid), expectOrdersDataCacheEmpty(removedOrderId), expectSubaccountsOrderIdsCacheEmpty(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.CANCELED), ]); expectNoWebsocketMessagesSent(producerSendSpy); expectTimingStats(true, true); @@ -1663,6 +1672,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheFound(expectedOrderUuid), expectOrdersDataCacheFound(removedOrderId), expectSubaccountsOrderIdsCacheFound(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.NOT_CANCELED), ]); expectTimingStats(false, false, false, false, true); @@ -1725,6 +1735,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheFound(expectedOrderUuid), expectOrdersDataCacheFound(removedOrderId), expectSubaccountsOrderIdsCacheFound(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.NOT_CANCELED), ]); expectTimingStats(false, false, false, false, true, true); @@ -1804,6 +1815,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheFound(expectedOrderUuid), expectOrdersDataCacheFound(removedOrderId), expectSubaccountsOrderIdsCacheFound(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.NOT_CANCELED), ]); expectTimingStats(false, false, false, false, true, true); @@ -1860,6 +1872,7 @@ describe('OrderRemoveHandler', () => { expectOrdersCacheFound(expectedOrderUuid), expectOrdersDataCacheFound(removedOrderId), expectSubaccountsOrderIdsCacheFound(redisTestConstants.defaultSubaccountUuid), + expectCanceledOrderStatus(expectedOrderUuid, CanceledOrderStatus.NOT_CANCELED), ]); expectTimingStats(false, false, false, false, true, true); diff --git a/indexer/services/vulcan/__tests__/helpers/helpers.ts b/indexer/services/vulcan/__tests__/helpers/helpers.ts index f0b74080fb5..370949bdf05 100644 --- a/indexer/services/vulcan/__tests__/helpers/helpers.ts +++ b/indexer/services/vulcan/__tests__/helpers/helpers.ts @@ -5,6 +5,7 @@ import { redisTestConstants, OrderbookLevelsCache, CanceledOrdersCache, + CanceledOrderStatus, } from '@dydxprotocol-indexer/redis'; import { OffChainUpdateV1 } from '@dydxprotocol-indexer/v4-protos'; import { KafkaMessage } from 'kafkajs'; @@ -80,16 +81,11 @@ export function setTransactionHash( return messageWithTxhash; } -export async function expectCanceledOrdersCacheFound( +export async function expectCanceledOrderStatus( orderId: string, -): Promise { - const orderExists: boolean = await CanceledOrdersCache.isOrderCanceled(orderId, redisClient); - expect(orderExists).toEqual(true); -} - -export async function expectCanceledOrdersCacheEmpty( - orderId: string, -): Promise { - const orderExists: boolean = await CanceledOrdersCache.isOrderCanceled(orderId, redisClient); - expect(orderExists).toEqual(false); + canceledOrderStatus: CanceledOrderStatus, +) { + expect(await CanceledOrdersCache.getOrderCanceledStatus(orderId, redisClient)).toEqual( + canceledOrderStatus, + ); } diff --git a/indexer/services/vulcan/src/handlers/order-place-handler.ts b/indexer/services/vulcan/src/handlers/order-place-handler.ts index 7514d8bac97..9fbe3665684 100644 --- a/indexer/services/vulcan/src/handlers/order-place-handler.ts +++ b/indexer/services/vulcan/src/handlers/order-place-handler.ts @@ -364,7 +364,7 @@ export class OrderPlaceHandler extends Handler { orderId: string, ): Promise { await runFuncWithTimingStat( - CanceledOrdersCache.removeOrderFromCache(orderId, redisClient), + CanceledOrdersCache.removeOrderFromCaches(orderId, redisClient), this.generateTimingStatsOptions('remove_order_from_cancel_cache'), ); } diff --git a/indexer/services/vulcan/src/handlers/order-remove-handler.ts b/indexer/services/vulcan/src/handlers/order-remove-handler.ts index 9bcd27dd532..fcf20837aea 100644 --- a/indexer/services/vulcan/src/handlers/order-remove-handler.ts +++ b/indexer/services/vulcan/src/handlers/order-remove-handler.ts @@ -300,7 +300,7 @@ export class OrderRemoveHandler extends Handler { } // TODO: consolidate remove handler logic into a single lua script. await this.addOrderToCanceledOrdersCache( - OrderTable.orderIdToUuid(orderRemove.removedOrderId!), + orderRemove, Date.now(), ); } @@ -338,13 +338,28 @@ export class OrderRemoveHandler extends Handler { * @protected */ protected async addOrderToCanceledOrdersCache( - orderId: string, + orderRemove: OrderRemoveV1, timestampMs: number, ): Promise { - await runFuncWithTimingStat( - CanceledOrdersCache.addCanceledOrderId(orderId, timestampMs, redisClient), - this.generateTimingStatsOptions('add_order_to_canceled_order_cache'), - ); + const orderId: string = OrderTable.orderIdToUuid(orderRemove.removedOrderId!); + + if ( + orderRemove.removalStatus === + OrderRemoveV1_OrderRemovalStatus.ORDER_REMOVAL_STATUS_BEST_EFFORT_CANCELED + ) { + await runFuncWithTimingStat( + CanceledOrdersCache.addBestEffortCanceledOrderId(orderId, timestampMs, redisClient), + this.generateTimingStatsOptions('add_order_to_canceled_order_cache'), + ); + } else if ( + orderRemove.removalStatus === + OrderRemoveV1_OrderRemovalStatus.ORDER_REMOVAL_STATUS_CANCELED + ) { + await runFuncWithTimingStat( + CanceledOrdersCache.addCanceledOrderId(orderId, timestampMs, redisClient), + this.generateTimingStatsOptions('add_order_to_canceled_order_cache'), + ); + } } /**