feat: VIP routing in the router, don't create fallback orders for native [TKR-243] (#440)
* feat: calculate all routes and VIP only routes in a single router call * fix: Try to disable using fallback orders for quotes with native orders * fix: create VIP sources set once per routing call * chore: use private publish version of neon-router to test * chore(lint): comment out unused fn _addOptionalFallbackAsync * chore: update to latest private publish of neon-router * refactor: fix router metrics beforeTimeMs naming * feat: don't recompute isVip for ever sample of a source * chore: update neon-router to real published version * fix: merge conflict resolution issue * chore: add asset-swapper changelog entry
This commit is contained in:
Kim Persson
GitHub
parent
f6e85aedf1
commit
4cd767ecb8
@ -5,6 +5,10 @@
|
||||
{
|
||||
"note": "Routing glue optimization",
|
||||
"pr": 439
|
||||
},
|
||||
{
|
||||
"note": "Move VIP source routing into neon-router & disable fallback orders for native/plp",
|
||||
"pr": 440
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@ -66,7 +66,7 @@
|
||||
"@0x/contracts-zero-ex": "^0.31.1",
|
||||
"@0x/dev-utils": "^4.2.11",
|
||||
"@0x/json-schemas": "^6.4.1",
|
||||
"@0x/neon-router": "^0.3.3",
|
||||
"@0x/neon-router": "^0.3.5",
|
||||
"@0x/protocol-utils": "^1.11.1",
|
||||
"@0x/quote-server": "^6.0.6",
|
||||
"@0x/types": "^3.3.4",
|
||||
|
||||
@ -570,7 +570,10 @@ export class MarketOperationUtils {
|
||||
}
|
||||
|
||||
// Generate a fallback path if required
|
||||
await this._addOptionalFallbackAsync(side, inputAmount, optimalPath, dexQuotes, fills, opts, penaltyOpts);
|
||||
// TODO(kimpers): Will experiment with disabling this and see how it affects revert rate
|
||||
// to avoid yet another router roundtrip
|
||||
// TODO: clean this up if we don't need it
|
||||
// await this._addOptionalFallbackAsync(side, inputAmount, optimalPath, dexQuotes, fills, opts, penaltyOpts);
|
||||
const collapsedPath = optimalPath.collapse(orderOpts);
|
||||
|
||||
return {
|
||||
@ -774,6 +777,8 @@ export class MarketOperationUtils {
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
* TODO(kimpers): Remove this when we know that it's safe to drop the fallbacks on native orders
|
||||
// tslint:disable-next-line: prefer-function-over-method
|
||||
private async _addOptionalFallbackAsync(
|
||||
side: MarketOperation,
|
||||
@ -839,6 +844,7 @@ export class MarketOperationUtils {
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
// tslint:disable: max-file-line-count
|
||||
|
||||
@ -76,7 +76,8 @@ function findRoutesAndCreateOptimalPath(
|
||||
opts: PathPenaltyOpts,
|
||||
fees: FeeSchedule,
|
||||
neonRouterNumSamples: number,
|
||||
): Path | undefined {
|
||||
vipSourcesSet: Set<ERC20BridgeSource>,
|
||||
): { allSourcesPath: Path | undefined; vipSourcesPath: Path | undefined } | undefined {
|
||||
// Currently the rust router is unable to handle 1 base unit sized quotes and will error out
|
||||
// To avoid flooding the logs with these errors we just return an insufficient liquidity error
|
||||
// which is how the JS router handles these quotes today
|
||||
@ -94,6 +95,127 @@ function findRoutesAndCreateOptimalPath(
|
||||
return fills[0];
|
||||
};
|
||||
|
||||
const createPathFromStrategy = (sourcesRustRoute: Float64Array, sourcesOutputAmounts: Float64Array) => {
|
||||
const routesAndSamplesAndOutputs = _.zip(
|
||||
sourcesRustRoute,
|
||||
samplesAndNativeOrdersWithResults,
|
||||
sourcesOutputAmounts,
|
||||
sampleSourcePathIds,
|
||||
);
|
||||
const adjustedFills: Fill[] = [];
|
||||
const totalRoutedAmount = BigNumber.sum(...sourcesRustRoute);
|
||||
|
||||
const scale = input.dividedBy(totalRoutedAmount);
|
||||
for (const [
|
||||
routeInput,
|
||||
routeSamplesAndNativeOrders,
|
||||
outputAmount,
|
||||
sourcePathId,
|
||||
] of routesAndSamplesAndOutputs) {
|
||||
if (!Number.isFinite(outputAmount)) {
|
||||
DEFAULT_WARNING_LOGGER(rustArgs, `neon-router: invalid route outputAmount ${outputAmount}`);
|
||||
return undefined;
|
||||
}
|
||||
if (!routeInput || !routeSamplesAndNativeOrders || !outputAmount) {
|
||||
continue;
|
||||
}
|
||||
// TODO(kimpers): [TKR-241] amounts are sometimes clipped in the router due to precision loss for number/f64
|
||||
// we can work around it by scaling it and rounding up. However now we end up with a total amount of a couple base units too much
|
||||
const rustInputAdjusted = BigNumber.min(
|
||||
new BigNumber(routeInput).multipliedBy(scale).integerValue(BigNumber.ROUND_CEIL),
|
||||
input,
|
||||
);
|
||||
|
||||
const current = routeSamplesAndNativeOrders[routeSamplesAndNativeOrders.length - 1];
|
||||
if (!isDexSample(current)) {
|
||||
const nativeFill = nativeOrdersToFills(
|
||||
side,
|
||||
[current],
|
||||
rustInputAdjusted,
|
||||
opts.outputAmountPerEth,
|
||||
opts.inputAmountPerEth,
|
||||
fees,
|
||||
false,
|
||||
)[0] as Fill | undefined;
|
||||
// Note: If the order has an adjusted rate of less than or equal to 0 it will be skipped
|
||||
// and nativeFill will be `undefined`
|
||||
if (nativeFill) {
|
||||
// NOTE: For Limit/RFQ orders we are done here. No need to scale output
|
||||
adjustedFills.push({ ...nativeFill, sourcePathId: sourcePathId ?? hexUtils.random() });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// NOTE: For DexSamples only
|
||||
let fill = createFill(current);
|
||||
if (!fill) {
|
||||
continue;
|
||||
}
|
||||
const routeSamples = routeSamplesAndNativeOrders as Array<DexSample<FillData>>;
|
||||
// Descend to approach a closer fill for fillData which may not be consistent
|
||||
// throughout the path (UniswapV3) and for a closer guesstimate at
|
||||
// gas used
|
||||
|
||||
assert.assert(routeSamples.length >= 1, 'Found no sample to use for source');
|
||||
for (let k = routeSamples.length - 1; k >= 0; k--) {
|
||||
if (k === 0) {
|
||||
fill = createFill(routeSamples[0]) ?? fill;
|
||||
}
|
||||
if (rustInputAdjusted.isGreaterThan(routeSamples[k].input)) {
|
||||
const left = routeSamples[k];
|
||||
const right = routeSamples[k + 1];
|
||||
if (left && right) {
|
||||
fill =
|
||||
createFill({
|
||||
...right, // default to the greater (for gas used)
|
||||
input: rustInputAdjusted,
|
||||
output: new BigNumber(outputAmount),
|
||||
}) ?? fill;
|
||||
} else {
|
||||
assert.assert(Boolean(left || right), 'No valid sample to use');
|
||||
fill = createFill(left || right) ?? fill;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(kimpers): remove once we have solved the rounding/precision loss issues in the Rust router
|
||||
const maxSampledOutput = BigNumber.max(...routeSamples.map(s => s.output));
|
||||
// Scale output by scale factor but never go above the largest sample in sell quotes (unknown liquidity) or below 1 base unit (unfillable)
|
||||
const scaleOutput = (output: BigNumber) => {
|
||||
// Don't try to scale 0 output as it will be clamped to 1
|
||||
if (output.eq(ZERO_AMOUNT)) {
|
||||
return output;
|
||||
}
|
||||
|
||||
const scaled = output
|
||||
.times(scale)
|
||||
.decimalPlaces(0, side === MarketOperation.Sell ? BigNumber.ROUND_FLOOR : BigNumber.ROUND_CEIL);
|
||||
const capped = MarketOperation.Sell ? BigNumber.min(scaled, maxSampledOutput) : scaled;
|
||||
|
||||
return BigNumber.max(capped, 1);
|
||||
};
|
||||
|
||||
adjustedFills.push({
|
||||
...fill,
|
||||
input: rustInputAdjusted,
|
||||
output: scaleOutput(fill.output),
|
||||
adjustedOutput: scaleOutput(fill.adjustedOutput),
|
||||
index: 0,
|
||||
parent: undefined,
|
||||
sourcePathId: sourcePathId ?? hexUtils.random(),
|
||||
});
|
||||
}
|
||||
|
||||
if (adjustedFills.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const pathFromRustInputs = Path.create(side, adjustedFills, input, opts);
|
||||
|
||||
return pathFromRustInputs;
|
||||
};
|
||||
|
||||
const samplesAndNativeOrdersWithResults: Array<DexSample[] | NativeOrderWithFillableAmounts[]> = [];
|
||||
const serializedPaths: SerializedPath[] = [];
|
||||
const sampleSourcePathIds: string[] = [];
|
||||
@ -136,6 +258,7 @@ function findRoutesAndCreateOptimalPath(
|
||||
inputs: [],
|
||||
outputs: [],
|
||||
outputFees: [],
|
||||
isVip: vipSourcesSet.has(singleSourceSamplesWithOutput[0]?.source),
|
||||
},
|
||||
);
|
||||
|
||||
@ -194,6 +317,7 @@ function findRoutesAndCreateOptimalPath(
|
||||
inputs,
|
||||
outputs,
|
||||
outputFees,
|
||||
isVip: true,
|
||||
};
|
||||
|
||||
samplesAndNativeOrdersWithResults.push([nativeOrder]);
|
||||
@ -212,130 +336,42 @@ function findRoutesAndCreateOptimalPath(
|
||||
};
|
||||
|
||||
const allSourcesRustRoute = new Float64Array(rustArgs.pathsIn.length);
|
||||
const strategySourcesOutputAmounts = new Float64Array(rustArgs.pathsIn.length);
|
||||
route(rustArgs, allSourcesRustRoute, strategySourcesOutputAmounts, neonRouterNumSamples);
|
||||
const allSourcesOutputAmounts = new Float64Array(rustArgs.pathsIn.length);
|
||||
const vipSourcesRustRoute = new Float64Array(rustArgs.pathsIn.length);
|
||||
const vipSourcesOutputAmounts = new Float64Array(rustArgs.pathsIn.length);
|
||||
|
||||
route(
|
||||
rustArgs,
|
||||
allSourcesRustRoute,
|
||||
allSourcesOutputAmounts,
|
||||
vipSourcesRustRoute,
|
||||
vipSourcesOutputAmounts,
|
||||
neonRouterNumSamples,
|
||||
);
|
||||
assert.assert(
|
||||
rustArgs.pathsIn.length === allSourcesRustRoute.length,
|
||||
'different number of sources in the Router output than the input',
|
||||
);
|
||||
assert.assert(
|
||||
rustArgs.pathsIn.length === strategySourcesOutputAmounts.length,
|
||||
rustArgs.pathsIn.length === allSourcesOutputAmounts.length,
|
||||
'different number of sources in the Router output amounts results than the input',
|
||||
);
|
||||
assert.assert(
|
||||
rustArgs.pathsIn.length === vipSourcesRustRoute.length,
|
||||
'different number of sources in the Router output than the input',
|
||||
);
|
||||
assert.assert(
|
||||
rustArgs.pathsIn.length === vipSourcesOutputAmounts.length,
|
||||
'different number of sources in the Router output amounts results than the input',
|
||||
);
|
||||
|
||||
const routesAndSamplesAndOutputs = _.zip(
|
||||
allSourcesRustRoute,
|
||||
samplesAndNativeOrdersWithResults,
|
||||
strategySourcesOutputAmounts,
|
||||
sampleSourcePathIds,
|
||||
);
|
||||
const adjustedFills: Fill[] = [];
|
||||
const totalRoutedAmount = BigNumber.sum(...allSourcesRustRoute);
|
||||
const allSourcesPath = createPathFromStrategy(allSourcesRustRoute, allSourcesOutputAmounts);
|
||||
const vipSourcesPath = createPathFromStrategy(vipSourcesRustRoute, vipSourcesOutputAmounts);
|
||||
|
||||
const scale = input.dividedBy(totalRoutedAmount);
|
||||
for (const [routeInput, routeSamplesAndNativeOrders, outputAmount, sourcePathId] of routesAndSamplesAndOutputs) {
|
||||
if (!Number.isFinite(outputAmount)) {
|
||||
DEFAULT_WARNING_LOGGER(rustArgs, `neon-router: invalid route outputAmount ${outputAmount}`);
|
||||
return undefined;
|
||||
}
|
||||
if (!routeInput || !routeSamplesAndNativeOrders || !outputAmount) {
|
||||
continue;
|
||||
}
|
||||
// TODO(kimpers): [TKR-241] amounts are sometimes clipped in the router due to precision loss for number/f64
|
||||
// we can work around it by scaling it and rounding up. However now we end up with a total amount of a couple base units too much
|
||||
const rustInputAdjusted = BigNumber.min(
|
||||
new BigNumber(routeInput).multipliedBy(scale).integerValue(BigNumber.ROUND_CEIL),
|
||||
input,
|
||||
);
|
||||
|
||||
const current = routeSamplesAndNativeOrders[routeSamplesAndNativeOrders.length - 1];
|
||||
if (!isDexSample(current)) {
|
||||
const nativeFill = nativeOrdersToFills(
|
||||
side,
|
||||
[current],
|
||||
rustInputAdjusted,
|
||||
opts.outputAmountPerEth,
|
||||
opts.inputAmountPerEth,
|
||||
fees,
|
||||
false,
|
||||
)[0] as Fill | undefined;
|
||||
// Note: If the order has an adjusted rate of less than or equal to 0 it will be skipped
|
||||
// and nativeFill will be `undefined`
|
||||
if (nativeFill) {
|
||||
// NOTE: For Limit/RFQ orders we are done here. No need to scale output
|
||||
adjustedFills.push({ ...nativeFill, sourcePathId: sourcePathId ?? hexUtils.random() });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// NOTE: For DexSamples only
|
||||
let fill = createFill(current);
|
||||
if (!fill) {
|
||||
continue;
|
||||
}
|
||||
const routeSamples = routeSamplesAndNativeOrders as Array<DexSample<FillData>>;
|
||||
// Descend to approach a closer fill for fillData which may not be consistent
|
||||
// throughout the path (UniswapV3) and for a closer guesstimate at
|
||||
// gas used
|
||||
|
||||
assert.assert(routeSamples.length >= 1, 'Found no sample to use for source');
|
||||
for (let k = routeSamples.length - 1; k >= 0; k--) {
|
||||
if (k === 0) {
|
||||
fill = createFill(routeSamples[0]) ?? fill;
|
||||
}
|
||||
if (rustInputAdjusted.isGreaterThan(routeSamples[k].input)) {
|
||||
const left = routeSamples[k];
|
||||
const right = routeSamples[k + 1];
|
||||
if (left && right) {
|
||||
fill =
|
||||
createFill({
|
||||
...right, // default to the greater (for gas used)
|
||||
input: rustInputAdjusted,
|
||||
output: new BigNumber(outputAmount),
|
||||
}) ?? fill;
|
||||
} else {
|
||||
assert.assert(Boolean(left || right), 'No valid sample to use');
|
||||
fill = createFill(left || right) ?? fill;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(kimpers): remove once we have solved the rounding/precision loss issues in the Rust router
|
||||
const maxSampledOutput = BigNumber.max(...routeSamples.map(s => s.output));
|
||||
// Scale output by scale factor but never go above the largest sample in sell quotes (unknown liquidity) or below 1 base unit (unfillable)
|
||||
const scaleOutput = (output: BigNumber) => {
|
||||
// Don't try to scale 0 output as it will be clamped to 1
|
||||
if (output.eq(ZERO_AMOUNT)) {
|
||||
return output;
|
||||
}
|
||||
|
||||
const scaled = output
|
||||
.times(scale)
|
||||
.decimalPlaces(0, side === MarketOperation.Sell ? BigNumber.ROUND_FLOOR : BigNumber.ROUND_CEIL);
|
||||
const capped = MarketOperation.Sell ? BigNumber.min(scaled, maxSampledOutput) : scaled;
|
||||
|
||||
return BigNumber.max(capped, 1);
|
||||
};
|
||||
|
||||
adjustedFills.push({
|
||||
...fill,
|
||||
input: rustInputAdjusted,
|
||||
output: scaleOutput(fill.output),
|
||||
adjustedOutput: scaleOutput(fill.adjustedOutput),
|
||||
index: 0,
|
||||
parent: undefined,
|
||||
sourcePathId: sourcePathId ?? hexUtils.random(),
|
||||
});
|
||||
}
|
||||
|
||||
if (adjustedFills.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const pathFromRustInputs = Path.create(side, adjustedFills, input, opts);
|
||||
|
||||
return pathFromRustInputs;
|
||||
return {
|
||||
allSourcesPath,
|
||||
vipSourcesPath,
|
||||
};
|
||||
}
|
||||
|
||||
export function findOptimalRustPathFromSamples(
|
||||
@ -349,9 +385,18 @@ export function findOptimalRustPathFromSamples(
|
||||
neonRouterNumSamples: number,
|
||||
samplerMetrics?: SamplerMetrics,
|
||||
): Path | undefined {
|
||||
const beforeAllTimeMs = performance.now();
|
||||
let beforeTimeMs = performance.now();
|
||||
const allSourcesPath = findRoutesAndCreateOptimalPath(
|
||||
const beforeTimeMs = performance.now();
|
||||
const sendMetrics = () => {
|
||||
// tslint:disable-next-line: no-unused-expression
|
||||
samplerMetrics &&
|
||||
samplerMetrics.logRouterDetails({
|
||||
router: 'neon-router',
|
||||
type: 'total',
|
||||
timingMs: performance.now() - beforeTimeMs,
|
||||
});
|
||||
};
|
||||
const vipSourcesSet = new Set(VIP_ERC20_BRIDGE_SOURCES_BY_CHAIN_ID[chainId]);
|
||||
const paths = findRoutesAndCreateOptimalPath(
|
||||
side,
|
||||
samples,
|
||||
nativeOrders,
|
||||
@ -359,58 +404,22 @@ export function findOptimalRustPathFromSamples(
|
||||
opts,
|
||||
fees,
|
||||
neonRouterNumSamples,
|
||||
vipSourcesSet,
|
||||
);
|
||||
// tslint:disable-next-line: no-unused-expression
|
||||
samplerMetrics &&
|
||||
samplerMetrics.logRouterDetails({
|
||||
router: 'neon-router',
|
||||
type: 'all',
|
||||
timingMs: performance.now() - beforeTimeMs,
|
||||
});
|
||||
if (!allSourcesPath) {
|
||||
|
||||
if (!paths) {
|
||||
sendMetrics();
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const vipSources = VIP_ERC20_BRIDGE_SOURCES_BY_CHAIN_ID[chainId];
|
||||
const { allSourcesPath, vipSourcesPath } = paths;
|
||||
|
||||
// HACK(kimpers): The Rust router currently doesn't account for VIP sources correctly
|
||||
// we need to try to route them in isolation and compare with the results all sources
|
||||
if (vipSources.length > 0) {
|
||||
beforeTimeMs = performance.now();
|
||||
const vipSourcesSet = new Set(vipSources);
|
||||
const vipSourcesSamples = samples.filter(s => s[0] && vipSourcesSet.has(s[0].source));
|
||||
|
||||
if (vipSourcesSamples.length > 0) {
|
||||
const vipSourcesPath = findRoutesAndCreateOptimalPath(
|
||||
side,
|
||||
vipSourcesSamples,
|
||||
[],
|
||||
input,
|
||||
opts,
|
||||
fees,
|
||||
neonRouterNumSamples,
|
||||
);
|
||||
// tslint:disable-next-line: no-unused-expression
|
||||
samplerMetrics &&
|
||||
samplerMetrics.logRouterDetails({
|
||||
router: 'neon-router',
|
||||
type: 'vip',
|
||||
timingMs: performance.now() - beforeTimeMs,
|
||||
});
|
||||
|
||||
if (vipSourcesPath?.isBetterThan(allSourcesPath)) {
|
||||
return vipSourcesPath;
|
||||
}
|
||||
}
|
||||
if (!allSourcesPath || vipSourcesPath?.isBetterThan(allSourcesPath)) {
|
||||
sendMetrics();
|
||||
return vipSourcesPath;
|
||||
}
|
||||
// tslint:disable-next-line: no-unused-expression
|
||||
samplerMetrics &&
|
||||
samplerMetrics.logRouterDetails({
|
||||
router: 'neon-router',
|
||||
type: 'total',
|
||||
timingMs: performance.now() - beforeAllTimeMs,
|
||||
});
|
||||
|
||||
sendMetrics();
|
||||
return allSourcesPath;
|
||||
}
|
||||
|
||||
|
||||
@ -952,9 +952,10 @@
|
||||
typedoc "~0.16.11"
|
||||
yargs "^10.0.3"
|
||||
|
||||
"@0x/neon-router@^0.3.3":
|
||||
version "0.3.3"
|
||||
resolved "https://registry.yarnpkg.com/@0x/neon-router/-/neon-router-0.3.3.tgz#dab540f4cd2aea6441ba29cbc35c28ca3f7a2b4f"
|
||||
"@0x/neon-router@^0.3.5":
|
||||
version "0.3.5"
|
||||
resolved "https://registry.yarnpkg.com/@0x/neon-router/-/neon-router-0.3.5.tgz#895e7a2dc65d492a413daaea283cbc0ca6df83fa"
|
||||
integrity sha512-8wizP3smc5o4jVg1smZzCCFo4ohOrgDhO4JFjF+/oNHbFImlGHOvmH9HQ2FJXAXiLEOTxrbp3T5XxP5GNATq3w==
|
||||
dependencies:
|
||||
"@mapbox/node-pre-gyp" "^1.0.5"
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user