From d06daf29577cc9a24aff8c0917a6b7bd2bf00ba9 Mon Sep 17 00:00:00 2001 From: Kim Persson Date: Mon, 4 Oct 2021 12:09:54 +0200 Subject: [PATCH] feat: initial integration of new router (#295) * feat: integrate Rust router with asset-swapper WIP * fix: produce outputFees in the format the Rust router expects * fix: correct output fee calc and only use the rust router for sells * fix: make sure numbers sent to the rust router are integers * hack: try to debug why rust router output is being overestimated WIP * refactor: clean up router debugging code * fix: don't use negative output fees for sells * feat: try VIP sources in isolation and compare with routing all sources * fix: adjust for FQT overhead when choosing between VIP, all sources WIP * fix: pass gasPrice to path_optimizer for EP overhead calculations * feat: buy support with the Rust Router WIP * chore: WIP commit trying to get buys working * refactor: use samples instead of fills for the Rust router * feat: add vip handling hack to sample based routing * fix: revert to 200 samplings for rust router when using pure samples * refactor: remove old hacky Path based Rust code, add back feature toggle * fix: scale both fill output and adjustedOutput my same factor as input * feat: initial plumbing for supporting RFQ/Limit orders * fix: incorrect bump of input amount by one base unit before routing * fix: add fake samples for rfq/limit orders to fulfill the 3 sample req * fix pass rfq orders in the correct format to the rust router * chore: remove debugging logs and clean up code & comments * fix: use published version of @0x/neon-router * hack: scale routed amounts to account for precision loss of number/f64 * refactor: clean up code and address initial review comments * fix: only remove trailing 0 output samples before passing to the router * refactor: consolidate eth to output token calc into ethToOutputAmount fn * fix: interpolate input between samples on output amount instead of price * fix: return no path when we have no samples, add sanity asserts * refactor: fix interpolation comment wording * fix: remove double adjusted source route input amount * chore: update changelog for asset-swapper --- packages/asset-swapper/CHANGELOG.json | 9 + packages/asset-swapper/package.json | 1 + packages/asset-swapper/src/swap_quoter.ts | 1 + packages/asset-swapper/src/types.ts | 2 +- .../utils/market_operation_utils/constants.ts | 19 +- .../src/utils/market_operation_utils/fills.ts | 42 +- .../src/utils/market_operation_utils/index.ts | 69 +++- .../src/utils/market_operation_utils/path.ts | 13 +- .../market_operation_utils/path_optimizer.ts | 363 +++++++++++++++++- .../src/utils/market_operation_utils/types.ts | 8 +- .../test/market_operation_utils_test.ts | 15 +- yarn.lock | 22 ++ 12 files changed, 528 insertions(+), 36 deletions(-) diff --git a/packages/asset-swapper/CHANGELOG.json b/packages/asset-swapper/CHANGELOG.json index aacb8b21c3..7eedd4d279 100644 --- a/packages/asset-swapper/CHANGELOG.json +++ b/packages/asset-swapper/CHANGELOG.json @@ -1,4 +1,13 @@ [ + { + "version": "16.29.0", + "changes": [ + { + "note": "Initial integration of neon-router (behind feature flag)", + "pr": 295 + } + ] + }, { "version": "16.28.0", "changes": [ diff --git a/packages/asset-swapper/package.json b/packages/asset-swapper/package.json index 15c53b978d..26a003faae 100644 --- a/packages/asset-swapper/package.json +++ b/packages/asset-swapper/package.json @@ -67,6 +67,7 @@ "@0x/dev-utils": "^4.2.9", "@0x/json-schemas": "^6.3.0", "@0x/protocol-utils": "^1.9.1", + "@0x/neon-router": "^0.1.3", "@0x/quote-server": "^6.0.6", "@0x/types": "^3.3.4", "@0x/typescript-typings": "^5.2.1", diff --git a/packages/asset-swapper/src/swap_quoter.ts b/packages/asset-swapper/src/swap_quoter.ts index 079a7371fc..54006ba846 100644 --- a/packages/asset-swapper/src/swap_quoter.ts +++ b/packages/asset-swapper/src/swap_quoter.ts @@ -363,6 +363,7 @@ export class SwapQuoter { const cloneOpts = _.omit(opts, 'gasPrice') as GetMarketOrdersOpts; const calcOpts: GetMarketOrdersOpts = { ...cloneOpts, + gasPrice, feeSchedule: _.mapValues(opts.feeSchedule, gasCost => (fillData: FillData) => gasCost === undefined ? 0 : gasPrice.times(gasCost(fillData)), ), diff --git a/packages/asset-swapper/src/types.ts b/packages/asset-swapper/src/types.ts index d7d250c19e..d23213013a 100644 --- a/packages/asset-swapper/src/types.ts +++ b/packages/asset-swapper/src/types.ts @@ -256,7 +256,7 @@ export interface RfqRequestOpts { /** * gasPrice: gas price to determine protocolFee amount, default to ethGasStation fast amount */ -export interface SwapQuoteRequestOpts extends GetMarketOrdersOpts { +export interface SwapQuoteRequestOpts extends Omit { gasPrice?: BigNumber; rfqt?: RfqRequestOpts; } diff --git a/packages/asset-swapper/src/utils/market_operation_utils/constants.ts b/packages/asset-swapper/src/utils/market_operation_utils/constants.ts index 72da15a81a..47fb8a7750 100644 --- a/packages/asset-swapper/src/utils/market_operation_utils/constants.ts +++ b/packages/asset-swapper/src/utils/market_operation_utils/constants.ts @@ -1637,6 +1637,23 @@ export const TRADER_JOE_ROUTER_BY_CHAIN_ID = valueByChainId( NULL_ADDRESS, ); +export const VIP_ERC20_BRIDGE_SOURCES_BY_CHAIN_ID = valueByChainId( + { + [ChainId.Mainnet]: [ERC20BridgeSource.UniswapV2, ERC20BridgeSource.SushiSwap, ERC20BridgeSource.UniswapV3], + [ChainId.BSC]: [ + ERC20BridgeSource.PancakeSwap, + ERC20BridgeSource.PancakeSwapV2, + ERC20BridgeSource.BakerySwap, + ERC20BridgeSource.SushiSwap, + ERC20BridgeSource.ApeSwap, + ERC20BridgeSource.CafeSwap, + ERC20BridgeSource.CheeseSwap, + ERC20BridgeSource.JulSwap, + ], + }, + [], +); + const uniswapV2CloneGasSchedule = (fillData?: FillData) => { // TODO: Different base cost if to/from ETH. let gas = 90e3; @@ -1779,7 +1796,7 @@ export const POSITIVE_SLIPPAGE_FEE_TRANSFORMER_GAS = new BigNumber(20000); // tslint:enable:custom-no-magic-numbers -export const DEFAULT_GET_MARKET_ORDERS_OPTS: GetMarketOrdersOpts = { +export const DEFAULT_GET_MARKET_ORDERS_OPTS: Omit = { // tslint:disable-next-line: custom-no-magic-numbers runLimit: 2 ** 15, excludedSources: [], diff --git a/packages/asset-swapper/src/utils/market_operation_utils/fills.ts b/packages/asset-swapper/src/utils/market_operation_utils/fills.ts index 4f800acc93..0cabbf240b 100644 --- a/packages/asset-swapper/src/utils/market_operation_utils/fills.ts +++ b/packages/asset-swapper/src/utils/market_operation_utils/fills.ts @@ -71,7 +71,25 @@ function hasLiquidity(fills: Fill[]): boolean { return true; } -function nativeOrdersToFills( +export function ethToOutputAmount({ + input, + output, + ethAmount, + inputAmountPerEth, + outputAmountPerEth, +}: { + input: BigNumber; + output: BigNumber; + inputAmountPerEth: BigNumber; + outputAmountPerEth: BigNumber; + ethAmount: BigNumber | number; +}): BigNumber { + return !outputAmountPerEth.isZero() + ? outputAmountPerEth.times(ethAmount) + : inputAmountPerEth.times(ethAmount).times(output.dividedToIntegerBy(input)); +} + +export function nativeOrdersToFills( side: MarketOperation, orders: NativeOrderWithFillableAmounts[], targetInput: BigNumber = POSITIVE_INF, @@ -89,9 +107,13 @@ function nativeOrdersToFills( const input = side === MarketOperation.Sell ? takerAmount : makerAmount; const output = side === MarketOperation.Sell ? makerAmount : takerAmount; const fee = fees[ERC20BridgeSource.Native] === undefined ? 0 : fees[ERC20BridgeSource.Native]!(o); - const outputPenalty = !outputAmountPerEth.isZero() - ? outputAmountPerEth.times(fee) - : inputAmountPerEth.times(fee).times(output.dividedToIntegerBy(input)); + const outputPenalty = ethToOutputAmount({ + input, + output, + inputAmountPerEth, + outputAmountPerEth, + ethAmount: fee, + }); // targetInput can be less than the order size // whilst the penalty is constant, it affects the adjusted output // only up until the target has been exhausted. @@ -132,7 +154,7 @@ function nativeOrdersToFills( return fills; } -function dexSamplesToFills( +export function dexSamplesToFills( side: MarketOperation, samples: DexSample[], outputAmountPerEth: BigNumber, @@ -156,9 +178,13 @@ function dexSamplesToFills( let penalty = ZERO_AMOUNT; if (i === 0) { // Only the first fill in a DEX path incurs a penalty. - penalty = !outputAmountPerEth.isZero() - ? outputAmountPerEth.times(fee) - : inputAmountPerEth.times(fee).times(output.dividedToIntegerBy(input)); + penalty = ethToOutputAmount({ + input, + output, + inputAmountPerEth, + outputAmountPerEth, + ethAmount: fee, + }); } const adjustedOutput = side === MarketOperation.Sell ? output.minus(penalty) : output.plus(penalty); diff --git a/packages/asset-swapper/src/utils/market_operation_utils/index.ts b/packages/asset-swapper/src/utils/market_operation_utils/index.ts index e96e0ea0c4..d3d3b58025 100644 --- a/packages/asset-swapper/src/utils/market_operation_utils/index.ts +++ b/packages/asset-swapper/src/utils/market_operation_utils/index.ts @@ -39,7 +39,7 @@ import { createFills } from './fills'; import { getBestTwoHopQuote } from './multihop_utils'; import { createOrdersFromTwoHopSample } from './orders'; import { Path, PathPenaltyOpts } from './path'; -import { fillsToSortedPaths, findOptimalPathAsync } from './path_optimizer'; +import { fillsToSortedPaths, findOptimalPathJSAsync, findOptimalRustPathFromSamples } from './path_optimizer'; import { DexOrderSampler, getSampleAmounts } from './sampler'; import { SourceFilters } from './source_filters'; import { @@ -48,7 +48,6 @@ import { DexSample, ERC20BridgeSource, Fill, - FillData, GenerateOptimizedOrdersOpts, GetMarketOrdersOpts, MarketSideLiquidity, @@ -57,6 +56,8 @@ import { OrderDomain, } from './types'; +const SHOULD_USE_RUST_ROUTER = process.env.RUST_ROUTER === 'true'; + // tslint:disable:boolean-naming export class MarketOperationUtils { @@ -326,12 +327,12 @@ export class MarketOperationUtils { public async getBatchMarketBuyOrdersAsync( batchNativeOrders: SignedNativeOrder[][], makerAmounts: BigNumber[], - opts?: Partial, + opts: Partial & { gasPrice: BigNumber }, ): Promise> { if (batchNativeOrders.length === 0) { throw new Error(AggregationError.EmptyOrders); } - const _opts = { ...DEFAULT_GET_MARKET_ORDERS_OPTS, ...opts }; + const _opts: GetMarketOrdersOpts = { ...DEFAULT_GET_MARKET_ORDERS_OPTS, ...opts }; const requestFilters = new SourceFilters().exclude(_opts.excludedSources).include(_opts.includedSources); const quoteSourceFilters = this._buySources.merge(requestFilters); @@ -409,6 +410,7 @@ export class MarketOperationUtils { excludedSources: _opts.excludedSources, feeSchedule: _opts.feeSchedule, allowFallback: _opts.allowFallback, + gasPrice: _opts.gasPrice, }, ); return optimizerResult; @@ -475,6 +477,7 @@ export class MarketOperationUtils { outputAmountPerEth, inputAmountPerEth, exchangeProxyOverhead: opts.exchangeProxyOverhead || (() => ZERO_AMOUNT), + gasPrice: opts.gasPrice, }; // NOTE: For sell quotes input is the taker asset and for buy quotes input is the maker asset @@ -485,8 +488,22 @@ export class MarketOperationUtils { const _unoptimizedPath = fillsToSortedPaths(fills, side, inputAmount, penaltyOpts)[0]; const unoptimizedPath = _unoptimizedPath ? _unoptimizedPath.collapse(orderOpts) : undefined; - // Find the optimal path - const optimalPath = await findOptimalPathAsync(side, fills, inputAmount, opts.runLimit, penaltyOpts); + // Find the optimal path using Rust router if enabled, otherwise fallback to JS Router + let optimalPath: Path | undefined; + if (SHOULD_USE_RUST_ROUTER) { + optimalPath = findOptimalRustPathFromSamples( + side, + dexQuotes, + [...nativeOrders, ...augmentedRfqtIndicativeQuotes], + inputAmount, + penaltyOpts, + opts.feeSchedule, + this._sampler.chainId, + ); + } else { + optimalPath = await findOptimalPathJSAsync(side, fills, inputAmount, opts.runLimit, penaltyOpts); + } + const optimalPathRate = optimalPath ? optimalPath.adjustedRate() : ZERO_AMOUNT; const { adjustedRate: bestTwoHopRate, quote: bestTwoHopQuote } = getBestTwoHopQuote( @@ -514,7 +531,7 @@ export class MarketOperationUtils { } // Generate a fallback path if required - await this._addOptionalFallbackAsync(side, inputAmount, optimalPath, fills, opts, penaltyOpts); + await this._addOptionalFallbackAsync(side, inputAmount, optimalPath, dexQuotes, fills, opts, penaltyOpts); const collapsedPath = optimalPath.collapse(orderOpts); return { @@ -536,9 +553,9 @@ export class MarketOperationUtils { nativeOrders: SignedNativeOrder[], amount: BigNumber, side: MarketOperation, - opts?: Partial, + opts: Partial & { gasPrice: BigNumber }, ): Promise { - const _opts = { ...DEFAULT_GET_MARKET_ORDERS_OPTS, ...opts }; + const _opts: GetMarketOrdersOpts = { ...DEFAULT_GET_MARKET_ORDERS_OPTS, ...opts }; const optimizerOpts: GenerateOptimizedOrdersOpts = { bridgeSlippage: _opts.bridgeSlippage, maxFallbackSlippage: _opts.maxFallbackSlippage, @@ -546,6 +563,7 @@ export class MarketOperationUtils { feeSchedule: _opts.feeSchedule, allowFallback: _opts.allowFallback, exchangeProxyOverhead: _opts.exchangeProxyOverhead, + gasPrice: _opts.gasPrice, }; if (nativeOrders.length === 0) { @@ -711,7 +729,8 @@ export class MarketOperationUtils { side: MarketOperation, inputAmount: BigNumber, optimalPath: Path, - fills: Array>>, + dexQuotes: DexSample[][], + fills: Fill[][], opts: GenerateOptimizedOrdersOpts, penaltyOpts: PathPenaltyOpts, ): Promise { @@ -725,13 +744,37 @@ export class MarketOperationUtils { if (opts.allowFallback && fragileFills.length !== 0) { // We create a fallback path that is exclusive of Native liquidity // This is the optimal on-chain path for the entire input amount - const sturdyFills = fills.filter(p => p.length > 0 && !fragileSources.includes(p[0].source)); - const sturdyOptimalPath = await findOptimalPathAsync(side, sturdyFills, inputAmount, opts.runLimit, { + const sturdyPenaltyOpts = { ...penaltyOpts, exchangeProxyOverhead: (sourceFlags: bigint) => // tslint:disable-next-line: no-bitwise penaltyOpts.exchangeProxyOverhead(sourceFlags | optimalPath.sourceFlags), - }); + }; + + let sturdyOptimalPath: Path | undefined; + if (SHOULD_USE_RUST_ROUTER) { + const sturdySamples = dexQuotes.filter( + samples => samples.length > 0 && !fragileSources.includes(samples[0].source), + ); + sturdyOptimalPath = findOptimalRustPathFromSamples( + side, + sturdySamples, + [], + inputAmount, + sturdyPenaltyOpts, + opts.feeSchedule, + this._sampler.chainId, + ); + } else { + const sturdyFills = fills.filter(p => p.length > 0 && !fragileSources.includes(p[0].source)); + sturdyOptimalPath = await findOptimalPathJSAsync( + side, + sturdyFills, + inputAmount, + opts.runLimit, + sturdyPenaltyOpts, + ); + } // Calculate the slippage of on-chain sources compared to the most optimal path // if within an acceptable threshold we enable a fallback to prevent reverts if ( diff --git a/packages/asset-swapper/src/utils/market_operation_utils/path.ts b/packages/asset-swapper/src/utils/market_operation_utils/path.ts index bddbfd9c76..6cb7ba84fe 100644 --- a/packages/asset-swapper/src/utils/market_operation_utils/path.ts +++ b/packages/asset-swapper/src/utils/market_operation_utils/path.ts @@ -3,6 +3,7 @@ import { BigNumber } from '@0x/utils'; import { MarketOperation } from '../../types'; import { POSITIVE_INF, ZERO_AMOUNT } from './constants'; +import { ethToOutputAmount } from './fills'; import { createBridgeOrder, createNativeOptimizedOrder, CreateOrderFromPathOpts, getMakerTakerTokens } from './orders'; import { getCompleteRate, getRate } from './rate_utils'; import { @@ -25,12 +26,14 @@ export interface PathPenaltyOpts { outputAmountPerEth: BigNumber; inputAmountPerEth: BigNumber; exchangeProxyOverhead: ExchangeProxyOverhead; + gasPrice: BigNumber; } export const DEFAULT_PATH_PENALTY_OPTS: PathPenaltyOpts = { outputAmountPerEth: ZERO_AMOUNT, inputAmountPerEth: ZERO_AMOUNT, exchangeProxyOverhead: () => ZERO_AMOUNT, + gasPrice: ZERO_AMOUNT, }; export class Path { @@ -143,9 +146,13 @@ export class Path { const { input, output } = this._adjustedSize; const { exchangeProxyOverhead, outputAmountPerEth, inputAmountPerEth } = this.pathPenaltyOpts; const gasOverhead = exchangeProxyOverhead(this.sourceFlags); - const pathPenalty = !outputAmountPerEth.isZero() - ? outputAmountPerEth.times(gasOverhead) - : inputAmountPerEth.times(gasOverhead).times(output.dividedToIntegerBy(input)); + const pathPenalty = ethToOutputAmount({ + input, + output, + inputAmountPerEth, + outputAmountPerEth, + ethAmount: gasOverhead, + }); return { input, output: this.side === MarketOperation.Sell ? output.minus(pathPenalty) : output.plus(pathPenalty), diff --git a/packages/asset-swapper/src/utils/market_operation_utils/path_optimizer.ts b/packages/asset-swapper/src/utils/market_operation_utils/path_optimizer.ts index ad08d48eef..fc3e5136a7 100644 --- a/packages/asset-swapper/src/utils/market_operation_utils/path_optimizer.ts +++ b/packages/asset-swapper/src/utils/market_operation_utils/path_optimizer.ts @@ -1,20 +1,377 @@ +import { assert } from '@0x/assert'; +import { ChainId } from '@0x/contract-addresses'; +import { OptimizerCapture, route, SerializedPath } from '@0x/neon-router'; import { BigNumber } from '@0x/utils'; import * as _ from 'lodash'; +import { performance } from 'perf_hooks'; -import { MarketOperation } from '../../types'; +import { DEFAULT_INFO_LOGGER } from '../../constants'; +import { MarketOperation, NativeOrderWithFillableAmounts } from '../../types'; +import { VIP_ERC20_BRIDGE_SOURCES_BY_CHAIN_ID } from '../market_operation_utils/constants'; +import { dexSamplesToFills, ethToOutputAmount, nativeOrdersToFills } from './fills'; import { DEFAULT_PATH_PENALTY_OPTS, Path, PathPenaltyOpts } from './path'; -import { ERC20BridgeSource, Fill } from './types'; +import { getRate } from './rate_utils'; +import { DexSample, ERC20BridgeSource, FeeSchedule, Fill, FillData } from './types'; // tslint:disable: prefer-for-of custom-no-magic-numbers completed-docs no-bitwise const RUN_LIMIT_DECAY_FACTOR = 0.5; +const RUST_ROUTER_NUM_SAMPLES = 200; +const FILL_QUOTE_TRANSFORMER_GAS_OVERHEAD = new BigNumber(150e3); +// NOTE: The Rust router will panic with less than 3 samples +const MIN_NUM_SAMPLE_INPUTS = 3; + +const isDexSample = (obj: DexSample | NativeOrderWithFillableAmounts): obj is DexSample => !!(obj as DexSample).source; + +function nativeOrderToNormalizedAmounts( + side: MarketOperation, + nativeOrder: NativeOrderWithFillableAmounts, +): { input: BigNumber; output: BigNumber } { + const { fillableTakerAmount, fillableTakerFeeAmount, fillableMakerAmount } = nativeOrder; + const makerAmount = fillableMakerAmount; + const takerAmount = fillableTakerAmount.plus(fillableTakerFeeAmount); + const input = side === MarketOperation.Sell ? takerAmount : makerAmount; + const output = side === MarketOperation.Sell ? makerAmount : takerAmount; + + return { input, output }; +} + +function calculateOuputFee( + side: MarketOperation, + sampleOrNativeOrder: DexSample | NativeOrderWithFillableAmounts, + outputAmountPerEth: BigNumber, + inputAmountPerEth: BigNumber, + fees: FeeSchedule, +): BigNumber { + if (isDexSample(sampleOrNativeOrder)) { + const { input, output, source, fillData } = sampleOrNativeOrder; + const fee = fees[source]?.(fillData) || 0; + const outputFee = ethToOutputAmount({ + input, + output, + inputAmountPerEth, + outputAmountPerEth, + ethAmount: fee, + }); + return outputFee; + } else { + const { input, output } = nativeOrderToNormalizedAmounts(side, sampleOrNativeOrder); + const fee = fees[ERC20BridgeSource.Native]?.(sampleOrNativeOrder) || 0; + const outputFee = ethToOutputAmount({ + input, + output, + inputAmountPerEth, + outputAmountPerEth, + ethAmount: fee, + }); + return outputFee; + } +} + +// Use linear interpolation to approximate the output +// at a certain input somewhere between the two samples +// See https://en.wikipedia.org/wiki/Linear_interpolation +const interpolateOutputFromSamples = ( + left: { input: BigNumber; output: BigNumber }, + right: { input: BigNumber; output: BigNumber }, + targetInput: BigNumber, +): BigNumber => + left.output.plus( + right.output + .minus(left.output) + .dividedBy(right.input.minus(left.input)) + .times(targetInput.minus(left.input)), + ); + +function findRoutesAndCreateOptimalPath( + side: MarketOperation, + samples: DexSample[][], + nativeOrders: NativeOrderWithFillableAmounts[], + input: BigNumber, + opts: PathPenaltyOpts, + fees: FeeSchedule, +): Path | undefined { + const createFill = (sample: DexSample) => + dexSamplesToFills(side, [sample], opts.outputAmountPerEth, opts.inputAmountPerEth, fees)[0]; + // Track sample id's to integers (required by rust router) + const sampleIdLookup: { [key: string]: number } = {}; + let sampleIdCounter = 0; + const sampleToId = (source: ERC20BridgeSource, index: number): number => { + const key = `${source}-${index}`; + if (sampleIdLookup[key]) { + return sampleIdLookup[key]; + } else { + sampleIdLookup[key] = ++sampleIdCounter; + return sampleIdLookup[key]; + } + }; + + const samplesAndNativeOrdersWithResults: Array = []; + const serializedPaths: SerializedPath[] = []; + for (const singleSourceSamples of samples) { + if (singleSourceSamples.length === 0) { + continue; + } + + const singleSourceSamplesWithOutput = [...singleSourceSamples]; + for (let i = singleSourceSamples.length - 1; i >= 0; i--) { + if (singleSourceSamples[i].output.isZero()) { + // Remove trailing 0 output samples + singleSourceSamplesWithOutput.pop(); + } else { + break; + } + } + + if (singleSourceSamplesWithOutput.length < MIN_NUM_SAMPLE_INPUTS) { + continue; + } + + // TODO(kimpers): Do we need to handle 0 entries, from eg Kyber? + const serializedPath = singleSourceSamplesWithOutput.reduce( + (memo, sample, sampleIdx) => { + memo.ids.push(sampleToId(sample.source, sampleIdx)); + memo.inputs.push(sample.input.integerValue().toNumber()); + memo.outputs.push(sample.output.integerValue().toNumber()); + memo.outputFees.push( + calculateOuputFee(side, sample, opts.outputAmountPerEth, opts.inputAmountPerEth, fees) + .integerValue() + .toNumber(), + ); + + return memo; + }, + { + ids: [], + inputs: [], + outputs: [], + outputFees: [], + }, + ); + + samplesAndNativeOrdersWithResults.push(singleSourceSamplesWithOutput); + serializedPaths.push(serializedPath); + } + + for (const [idx, nativeOrder] of nativeOrders.entries()) { + const { input: normalizedOrderInput, output: normalizedOrderOutput } = nativeOrderToNormalizedAmounts( + side, + nativeOrder, + ); + // NOTE: skip dummy order created in swap_quoter + // TODO: remove dummy order and this logic once we don't need the JS router + if (normalizedOrderInput.isLessThanOrEqualTo(0) || normalizedOrderOutput.isLessThanOrEqualTo(0)) { + continue; + } + + // HACK: the router requires at minimum 3 samples as a basis for interpolation + const inputs = [ + 0, + normalizedOrderInput + .dividedBy(2) + .integerValue() + .toNumber(), + normalizedOrderInput.integerValue().toNumber(), + ]; + const outputs = [ + 0, + normalizedOrderOutput + .dividedBy(2) + .integerValue() + .toNumber(), + normalizedOrderOutput.integerValue().toNumber(), + ]; + // NOTE: same fee no matter if full or partial fill + const fee = calculateOuputFee(side, nativeOrder, opts.outputAmountPerEth, opts.inputAmountPerEth, fees) + .integerValue() + .toNumber(); + const outputFees = [fee, fee, fee]; + // NOTE: ids can be the same for all fake samples + const id = sampleToId(ERC20BridgeSource.Native, idx); + const ids = [id, id, id]; + + const serializedPath: SerializedPath = { + ids, + inputs, + outputs, + outputFees, + }; + + samplesAndNativeOrdersWithResults.push([nativeOrder]); + serializedPaths.push(serializedPath); + } + + if (serializedPaths.length === 0) { + return undefined; + } + + const rustArgs: OptimizerCapture = { + side, + targetInput: input.toNumber(), + pathsIn: serializedPaths, + }; + + const before = performance.now(); + const allSourcesRustRoute = route(rustArgs, RUST_ROUTER_NUM_SAMPLES); + DEFAULT_INFO_LOGGER( + { router: 'neon-router', performanceMs: performance.now() - before, type: 'real' }, + 'Rust router real routing performance', + ); + + assert.assert( + rustArgs.pathsIn.length === allSourcesRustRoute.length, + 'different number of sources in the Router output than the input', + ); + + const routesAndSamples = _.zip(allSourcesRustRoute, samplesAndNativeOrdersWithResults); + + const adjustedFills: Fill[] = []; + const totalRoutedAmount = BigNumber.sum(...allSourcesRustRoute); + + const scale = input.dividedBy(totalRoutedAmount); + for (const [routeInput, routeSamplesAndNativeOrders] of routesAndSamples) { + if (!routeInput || !routeSamplesAndNativeOrders) { + continue; + } + // TODO(kimpers): [TKR-241] amounts are sometimes clipped in the router due to precisions loss for number/f64 + // we can work around it by scaling it and rounding up. However now we end up with a total amount of a couple base units too much + const rustInputAdjusted = BigNumber.min( + new BigNumber(routeInput).multipliedBy(scale).integerValue(BigNumber.ROUND_CEIL), + input, + ); + + const current = routeSamplesAndNativeOrders[routeSamplesAndNativeOrders.length - 1]; + if (!isDexSample(current)) { + const nativeFill = nativeOrdersToFills( + side, + [current], + rustInputAdjusted, + opts.outputAmountPerEth, + opts.inputAmountPerEth, + fees, + )[0]; + // NOTE: For Limit/RFQ orders we are done here. No need to scale output + adjustedFills.push(nativeFill); + continue; + } + + // NOTE: For DexSamples only + let fill = createFill(current); + const routeSamples = routeSamplesAndNativeOrders as Array>; + // Descend to approach a closer fill for fillData which may not be consistent + // throughout the path (UniswapV3) and for a closer guesstimate at + // gas used + + assert.assert(routeSamples.length >= 1, 'Found no sample to use for source'); + for (let k = routeSamples.length - 1; k >= 0; k--) { + if (k === 0) { + fill = createFill(routeSamples[0]); + } + if (rustInputAdjusted.isGreaterThan(routeSamples[k].input)) { + // Between here and the previous fill + // HACK: Use the midpoint between the two + const left = routeSamples[k]; + const right = routeSamples[k + 1]; + if (left && right) { + // Approximate how much output we get for the input with the surrounding samples + const interpolatedOutput = interpolateOutputFromSamples( + left, + right, + rustInputAdjusted, + ).decimalPlaces(0, side === MarketOperation.Sell ? BigNumber.ROUND_FLOOR : BigNumber.ROUND_CEIL); + + fill = createFill({ + ...right, // default to the greater (for gas used) + input: rustInputAdjusted, + output: interpolatedOutput, + }); + } else { + assert.assert(Boolean(left || right), 'No valid sample to use'); + fill = createFill(left || right); + } + break; + } + } + + const scaleOutput = (output: BigNumber) => + output + .dividedBy(fill.input) + .times(rustInputAdjusted) + .decimalPlaces(0, side === MarketOperation.Sell ? BigNumber.ROUND_FLOOR : BigNumber.ROUND_CEIL); + adjustedFills.push({ + ...fill, + input: rustInputAdjusted, + output: scaleOutput(fill.output), + adjustedOutput: scaleOutput(fill.adjustedOutput), + index: 0, + parent: undefined, + }); + } + + const pathFromRustInputs = Path.create(side, adjustedFills, input); + + return pathFromRustInputs; +} + +export function findOptimalRustPathFromSamples( + side: MarketOperation, + samples: DexSample[][], + nativeOrders: NativeOrderWithFillableAmounts[], + input: BigNumber, + opts: PathPenaltyOpts, + fees: FeeSchedule, + chainId: ChainId, +): Path | undefined { + const before = performance.now(); + const logPerformance = () => + DEFAULT_INFO_LOGGER( + { router: 'neon-router', performanceMs: performance.now() - before, type: 'total' }, + 'Rust router total routing performance', + ); + + const allSourcesPath = findRoutesAndCreateOptimalPath(side, samples, nativeOrders, input, opts, fees); + if (!allSourcesPath) { + return undefined; + } + + const vipSources = VIP_ERC20_BRIDGE_SOURCES_BY_CHAIN_ID[chainId]; + + // HACK(kimpers): The Rust router currently doesn't account for VIP sources correctly + // we need to try to route them in isolation and compare with the results all sources + if (vipSources.length > 0) { + const vipSourcesSet = new Set(vipSources); + const vipSourcesSamples = samples.filter(s => s[0] && vipSourcesSet.has(s[0].source)); + + if (vipSourcesSamples.length > 0) { + const vipSourcesPath = findRoutesAndCreateOptimalPath(side, vipSourcesSamples, [], input, opts, fees); + + const { input: allSourcesInput, output: allSourcesOutput } = allSourcesPath.adjustedSize(); + // NOTE: For sell quotes input is the taker asset and for buy quotes input is the maker asset + const gasCostInWei = FILL_QUOTE_TRANSFORMER_GAS_OVERHEAD.times(opts.gasPrice); + const fqtOverheadInOutputToken = gasCostInWei.times(opts.outputAmountPerEth); + const outputWithFqtOverhead = + side === MarketOperation.Sell + ? allSourcesOutput.minus(fqtOverheadInOutputToken) + : allSourcesOutput.plus(fqtOverheadInOutputToken); + const allSourcesAdjustedRateWithFqtOverhead = getRate(side, allSourcesInput, outputWithFqtOverhead); + + if (vipSourcesPath?.adjustedRate().isGreaterThan(allSourcesAdjustedRateWithFqtOverhead)) { + logPerformance(); + return vipSourcesPath; + } + } + } + + logPerformance(); + return allSourcesPath; +} /** * Find the optimal mixture of fills that maximizes (for sells) or minimizes * (for buys) output, while meeting the input requirement. */ -export async function findOptimalPathAsync( +export async function findOptimalPathJSAsync( side: MarketOperation, fills: Fill[][], targetInput: BigNumber, diff --git a/packages/asset-swapper/src/utils/market_operation_utils/types.ts b/packages/asset-swapper/src/utils/market_operation_utils/types.ts index 3056c5c70c..3e99836a1e 100644 --- a/packages/asset-swapper/src/utils/market_operation_utils/types.ts +++ b/packages/asset-swapper/src/utils/market_operation_utils/types.ts @@ -455,6 +455,11 @@ export interface GetMarketOrdersOpts { * hopping to. E.g DAI->USDC via an adjacent token WETH */ tokenAdjacencyGraph: TokenAdjacencyGraph; + + /** + * Gas price to use for quote + */ + gasPrice: BigNumber; } /** @@ -534,10 +539,11 @@ export interface GenerateOptimizedOrdersOpts { bridgeSlippage?: number; maxFallbackSlippage?: number; excludedSources?: ERC20BridgeSource[]; - feeSchedule?: FeeSchedule; + feeSchedule: FeeSchedule; exchangeProxyOverhead?: ExchangeProxyOverhead; allowFallback?: boolean; shouldBatchBridgeOrders?: boolean; + gasPrice: BigNumber; } export interface ComparisonPrice { diff --git a/packages/asset-swapper/test/market_operation_utils_test.ts b/packages/asset-swapper/test/market_operation_utils_test.ts index c516e2bfb2..1261244d0d 100644 --- a/packages/asset-swapper/test/market_operation_utils_test.ts +++ b/packages/asset-swapper/test/market_operation_utils_test.ts @@ -79,7 +79,7 @@ async function getMarketSellOrdersAsync( utils: MarketOperationUtils, nativeOrders: SignedNativeOrder[], takerAmount: BigNumber, - opts?: Partial, + opts: Partial & { gasPrice: BigNumber }, ): Promise { return utils.getOptimizerResultAsync(nativeOrders, takerAmount, MarketOperation.Sell, opts); } @@ -96,7 +96,7 @@ async function getMarketBuyOrdersAsync( utils: MarketOperationUtils, nativeOrders: SignedNativeOrder[], makerAmount: BigNumber, - opts?: Partial, + opts: Partial & { gasPrice: BigNumber }, ): Promise { return utils.getOptimizerResultAsync(nativeOrders, makerAmount, MarketOperation.Buy, opts); } @@ -459,7 +459,7 @@ describe('MarketOperationUtils tests', () => { FILL_AMOUNT, _.times(NUM_SAMPLES, i => DEFAULT_RATES[ERC20BridgeSource.Native][i]), ); - const DEFAULT_OPTS: Partial = { + const DEFAULT_OPTS: Partial & { gasPrice: BigNumber } = { numSamples: NUM_SAMPLES, sampleDistributionBase: 1, bridgeSlippage: 0, @@ -468,6 +468,7 @@ describe('MarketOperationUtils tests', () => { allowFallback: false, gasSchedule: {}, feeSchedule: {}, + gasPrice: new BigNumber(30e9), }; beforeEach(() => { @@ -1229,6 +1230,7 @@ describe('MarketOperationUtils tests', () => { excludedSources: [], numSamples: 4, bridgeSlippage: 0, + gasPrice: new BigNumber(30e9), }, ); const result = ordersAndReport.optimizedOrders; @@ -1298,7 +1300,8 @@ describe('MarketOperationUtils tests', () => { FILL_AMOUNT, _.times(NUM_SAMPLES, () => DEFAULT_RATES[ERC20BridgeSource.Native][0]), ); - const DEFAULT_OPTS: Partial = { + const GAS_PRICE = new BigNumber(100e9); // 100 gwei + const DEFAULT_OPTS: Partial & { gasPrice: BigNumber } = { numSamples: NUM_SAMPLES, sampleDistributionBase: 1, bridgeSlippage: 0, @@ -1307,6 +1310,7 @@ describe('MarketOperationUtils tests', () => { allowFallback: false, gasSchedule: {}, feeSchedule: {}, + gasPrice: GAS_PRICE, }; beforeEach(() => { @@ -1626,11 +1630,10 @@ describe('MarketOperationUtils tests', () => { getMedianSellRate: createGetMedianSellRate(ETH_TO_TAKER_RATE), }); const optimizer = new MarketOperationUtils(MOCK_SAMPLER, contractAddresses, ORDER_DOMAIN); - const gasPrice = 100e9; // 100 gwei const exchangeProxyOverhead = (sourceFlags: bigint) => sourceFlags === SOURCE_FLAGS.LiquidityProvider ? constants.ZERO_AMOUNT - : new BigNumber(1.3e5).times(gasPrice); + : new BigNumber(1.3e5).times(GAS_PRICE); const improvedOrdersResponse = await optimizer.getOptimizerResultAsync( createOrdersFromSellRates(FILL_AMOUNT, rates[ERC20BridgeSource.Native]), FILL_AMOUNT, diff --git a/yarn.lock b/yarn.lock index 150efebfc1..c51adbc4c0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -959,6 +959,13 @@ typedoc "~0.16.11" yargs "^10.0.3" +"@0x/neon-router@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@0x/neon-router/-/neon-router-0.1.3.tgz#70da4c17ca4b59dfe8b5e539673e364a70e62ebd" + integrity sha512-EfdrG829NalYjAK5/nMTD6YyJQgUzgssL2Hvyphu1ugWxWlZ3QMM9qpZsKt82hUiyZT/64I4JJ3hkerMhTaHeg== + dependencies: + "@mapbox/node-pre-gyp" "^1.0.5" + "@0x/order-utils@^10.2.4", "@0x/order-utils@^10.4.28": version "10.4.28" resolved "https://registry.yarnpkg.com/@0x/order-utils/-/order-utils-10.4.28.tgz#c7b2f7d87a7f9834f9aa6186fbac68f32a05a81d" @@ -2473,6 +2480,21 @@ semver "^7.3.4" tar "^6.1.0" +"@mapbox/node-pre-gyp@^1.0.5": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.5.tgz#2a0b32fcb416fb3f2250fd24cb2a81421a4f5950" + integrity sha512-4srsKPXWlIxp5Vbqz5uLfBN+du2fJChBoYn/f2h991WLdk7jUvcSk/McVLSv/X+xQIPI8eGD5GjrnygdyHnhPA== + dependencies: + detect-libc "^1.0.3" + https-proxy-agent "^5.0.0" + make-dir "^3.1.0" + node-fetch "^2.6.1" + nopt "^5.0.0" + npmlog "^4.1.2" + rimraf "^3.0.2" + semver "^7.3.4" + tar "^6.1.0" + "@mrmlnc/readdir-enhanced@^2.2.1": version "2.2.1" resolved "https://registry.yarnpkg.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde"