Merge branch 'v2-prototype' into fix/contract-wrappers/exchangeTransferSimulator

* v2-prototype:
  Fix a bug in SolCompilerArtifacts adapter config overriding
  Increase timeout for contract migrations
  Remove some copy-paste code
  Await transactions in migrations
  Fix typos
  Await transactions in migrations
  Await fake transactions
  Fix a typo
  Implement SolidityProfiler & adapt sol-cov to work with Geth

# Conflicts:
#	packages/migrations/CHANGELOG.json

packages/contract-wrappers/test/global_hooks.ts

@@ -6,7 +6,7 @@ import { provider } from './utils/web3_wrapper';
 before('migrate contracts', async function(): Promise<void> {
     // HACK: Since the migrations take longer then our global mocha timeout limit
     // we manually increase it for this before hook.
-    const mochaTestTimeoutMs = 20000;
+    const mochaTestTimeoutMs = 50000;
     this.timeout(mochaTestTimeoutMs);
     const txDefaults = {
         gas: devConstants.GAS_LIMIT,

packages/contracts/package.json

@@ -18,6 +18,7 @@
         "test": "yarn run_mocha",
         "rebuild_and_test": "run-s build test",
         "test:coverage": "SOLIDITY_COVERAGE=true run-s build run_mocha coverage:report:text coverage:report:lcov",
+        "test:profiler": "SOLIDITY_PROFILER=true run-s build run_mocha profiler:report:html",
         "run_mocha": "mocha --require source-map-support/register 'lib/test/**/*.js' --timeout 100000 --bail --exit",
         "compile": "sol-compiler",
         "clean": "shx rm -rf lib src/generated_contract_wrappers",
@@ -26,6 +27,7 @@
         "lint": "tslint --project . --exclude **/src/contract_wrappers/**/* --exclude **/lib/**/*",
         "coverage:report:text": "istanbul report text",
         "coverage:report:html": "istanbul report html && open coverage/index.html",
+        "profiler:report:html": "istanbul report html && open coverage/index.html",
         "coverage:report:lcov": "istanbul report lcov",
         "test:circleci": "yarn test"
     },

packages/contracts/src/utils/coverage.ts

@@ -14,7 +14,8 @@ export const coverage = {
     _getCoverageSubprovider(): CoverageSubprovider {
         const defaultFromAddress = devConstants.TESTRPC_FIRST_ADDRESS;
         const solCompilerArtifactAdapter = new SolCompilerArtifactAdapter();
-        const subprovider = new CoverageSubprovider(solCompilerArtifactAdapter, defaultFromAddress);
+        const isVerbose = true;
+        const subprovider = new CoverageSubprovider(solCompilerArtifactAdapter, defaultFromAddress, isVerbose);
         return subprovider;
     },
 };

packages/contracts/src/utils/profiler.ts

@@ -0,0 +1,27 @@
+import { devConstants } from '@0xproject/dev-utils';
+import { ProfilerSubprovider, SolCompilerArtifactAdapter } from '@0xproject/sol-cov';
+import * as _ from 'lodash';
+
+let profilerSubprovider: ProfilerSubprovider;
+
+export const profiler = {
+    start(): void {
+        profiler.getProfilerSubproviderSingleton().start();
+    },
+    stop(): void {
+        profiler.getProfilerSubproviderSingleton().stop();
+    },
+    getProfilerSubproviderSingleton(): ProfilerSubprovider {
+        if (_.isUndefined(profilerSubprovider)) {
+            profilerSubprovider = profiler._getProfilerSubprovider();
+        }
+        return profilerSubprovider;
+    },
+    _getProfilerSubprovider(): ProfilerSubprovider {
+        const defaultFromAddress = devConstants.TESTRPC_FIRST_ADDRESS;
+        const solCompilerArtifactAdapter = new SolCompilerArtifactAdapter();
+        const isVerbose = true;
+        const subprovider = new ProfilerSubprovider(solCompilerArtifactAdapter, defaultFromAddress, isVerbose);
+        return subprovider;
+    },
+};

packages/contracts/src/utils/web3_wrapper.ts

@@ -1,8 +1,10 @@
 import { devConstants, env, EnvVars, web3Factory } from '@0xproject/dev-utils';
 import { prependSubprovider } from '@0xproject/subproviders';
+import { logUtils } from '@0xproject/utils';
 import { Web3Wrapper } from '@0xproject/web3-wrapper';
 
 import { coverage } from './coverage';
+import { profiler } from './profiler';
 
 enum ProviderType {
     Ganache = 'ganache',
@@ -45,9 +47,29 @@ const providerConfigs = testProvider === ProviderType.Ganache ? ganacheConfigs :
 
 export const provider = web3Factory.getRpcProvider(providerConfigs);
 const isCoverageEnabled = env.parseBoolean(EnvVars.SolidityCoverage);
+const isProfilerEnabled = env.parseBoolean(EnvVars.SolidityProfiler);
+if (isCoverageEnabled && isProfilerEnabled) {
+    throw new Error(
+        `Unfortunately for now you can't enable both coverage and profiler at the same time. They both use coverage.json file and there is no way to configure that.`,
+    );
+}
 if (isCoverageEnabled) {
     const coverageSubprovider = coverage.getCoverageSubproviderSingleton();
     prependSubprovider(provider, coverageSubprovider);
 }
+if (isProfilerEnabled) {
+    if (testProvider === ProviderType.Ganache) {
+        logUtils.warn(
+            "Gas costs in Ganache traces are incorrect and we don't recommend using it for profiling. Please switch to Geth",
+        );
+        process.exit(1);
+    }
+    const profilerSubprovider = profiler.getProfilerSubproviderSingleton();
+    logUtils.log(
+        "By default profilerSubprovider is stopped so that you don't get noise from setup code. Don't forget to start it before the code you want to profile and stop it afterwards",
+    );
+    profilerSubprovider.stop();
+    prependSubprovider(provider, profilerSubprovider);
+}
 
 export const web3Wrapper = new Web3Wrapper(provider);

packages/contracts/test/global_hooks.ts

@@ -1,10 +1,15 @@
 import { env, EnvVars } from '@0xproject/dev-utils';
 
 import { coverage } from '../src/utils/coverage';
+import { profiler } from '../src/utils/profiler';
 
 after('generate coverage report', async () => {
     if (env.parseBoolean(EnvVars.SolidityCoverage)) {
         const coverageSubprovider = coverage.getCoverageSubproviderSingleton();
         await coverageSubprovider.writeCoverageAsync();
     }
+    if (env.parseBoolean(EnvVars.SolidityProfiler)) {
+        const profilerSubprovider = profiler.getProfilerSubproviderSingleton();
+        await profilerSubprovider.writeProfilerOutputAsync();
+    }
 });

packages/dev-utils/CHANGELOG.json

@@ -5,6 +5,10 @@
             {
                 "note": "Add optional parameter shouldUseFakeGasEstimate to Web3Config",
                 "pr": 622
+            },
+            {
+                "note": "Add SolidityProfiler to EnvVars",
+                "pr": 675
             }
         ]
     },

packages/dev-utils/src/env.ts

@@ -3,6 +3,7 @@ import * as process from 'process';
 
 export enum EnvVars {
     SolidityCoverage = 'SOLIDITY_COVERAGE',
+    SolidityProfiler = 'SOLIDITY_PROFILER',
     VerboseGanache = 'VERBOSE_GANACHE',
 }
 

packages/ethereum-types/CHANGELOG.json

@@ -1,11 +1,21 @@
 [
     {
-        "version": "0.0.1",
+        "version": "0.0.3",
+        "changes": [
+            {
+                "note": "Add `TraceParams` interface for `debug_traceTransaction` parameters",
+                "pr": 675
+            }
+        ]
+    },
+    {
+        "version": "0.0.2",
         "changes": [
             {
                 "note": "Initial publish",
                 "pr": "642"
             }
-        ]
+        ],
+        "timestamp": 1527811200
     }
 ]

packages/ethereum-types/src/index.ts

@@ -279,3 +279,9 @@ export enum SolidityTypes {
 export interface TransactionReceiptWithDecodedLogs extends TransactionReceipt {
     logs: Array<LogWithDecodedArgs<DecodedLogArgs> | LogEntry>;
 }
+
+export interface TraceParams {
+    disableMemory?: boolean;
+    disableStack?: boolean;
+    disableStorage?: boolean;
+}

packages/metacoin/package.json

@@ -17,7 +17,7 @@
         "test": "yarn run_mocha",
         "rebuild_and_test": "run-s build test",
         "test:coverage": "SOLIDITY_COVERAGE=true run-s build run_mocha coverage:report:text coverage:report:lcov",
-        "run_mocha": "mocha --require source-map-support/register lib/test/**/*_test.js lib/test/global_hooks.js --bail --exit",
+        "run_mocha": "mocha --require source-map-support/register lib/test/**/*_test.js lib/test/global_hooks.js --bail --exit --timeout 10000",
         "generate_contract_wrappers": "abi-gen --abis 'artifacts/Metacoin.json' --template ../contract_templates/contract.handlebars --partials '../contract_templates/partials/**/*.handlebars' --output src/contract_wrappers --backend ethers",
         "coverage:report:text": "istanbul report text",
         "coverage:report:html": "istanbul report html && open coverage/index.html",

packages/migrations/CHANGELOG.json

@@ -3,7 +3,12 @@
         "version": "0.0.7",
         "changes": [
             {
-                "note": "Export ArtifactWriter class"
+                "note": "Export ArtifactWriter class",
+                "pr": 684
+            },
+            {
+                "note": "Use AssetProxyOwner instead of MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddress",
+                "pr": 675
             }
         ]
     },

packages/migrations/artifacts/2.0.0/IValidator.json

@@ -30,21 +30,7 @@
 				"stateMutability": "view",
 				"type": "function"
 			}
-		],
+		]
-		"evm": {
-			"bytecode": {
-				"linkReferences": {},
-				"object": "0x",
-				"opcodes": "",
-				"sourceMap": ""
-			},
-			"deployedBytecode": {
-				"linkReferences": {},
-				"object": "0x",
-				"opcodes": "",
-				"sourceMap": ""
-			}
-		}
 	},
 	"sources": {
 		"current/protocol/Exchange/interfaces/IValidator.sol": {
@@ -73,4 +59,4 @@
 		}
 	},
 	"networks": {}
-}
+}

packages/migrations/artifacts/2.0.0/IWallet.json

@@ -26,21 +26,7 @@
 				"stateMutability": "view",
 				"type": "function"
 			}
-		],
+		]
-		"evm": {
-			"bytecode": {
-				"linkReferences": {},
-				"object": "0x",
-				"opcodes": "",
-				"sourceMap": ""
-			},
-			"deployedBytecode": {
-				"linkReferences": {},
-				"object": "0x",
-				"opcodes": "",
-				"sourceMap": ""
-			}
-		}
 	},
 	"sources": {
 		"current/protocol/Exchange/interfaces/IWallet.sol": {
@@ -69,4 +55,4 @@
 		}
 	},
 	"networks": {}
-}
+}

packages/migrations/package.json

@@ -31,7 +31,7 @@
             "v1":
                 "artifacts/1.0.0/@(DummyERC20Token|TokenTransferProxy_v1|Exchange_v1|TokenRegistry|MultiSigWallet|MultiSigWalletWithTimeLock|MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddress|TokenRegistry|ZRXToken|WETH9).json",
             "v2":
-                "artifacts/2.0.0/@(ERC20Token|DummyERC20Token|ERC721Token|DummyERC721Token|ERC20Proxy|ERC721Proxy|Exchange|MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddress|ZRXToken|WETH9|IWallet|IValidator).json"
+                "artifacts/2.0.0/@(ERC20Token|DummyERC20Token|ERC721Token|DummyERC721Token|ERC20Proxy|ERC721Proxy|Exchange|AssetProxyOwner|ZRXToken|WETH9|IWallet|IValidator).json"
         }
     },
     "license": "Apache-2.0",

packages/migrations/src/v1/migration.ts

@@ -68,8 +68,13 @@ export const runV1MigrationsAsync = async (provider: Provider, artifactsDir: str
     artifactsWriter.saveArtifact(multiSig);
 
     const owner = accounts[0];
-    await tokenTransferProxy.addAuthorizedAddress.sendTransactionAsync(exchange.address, { from: owner });
+
-    await tokenTransferProxy.transferOwnership.sendTransactionAsync(multiSig.address, { from: owner });
+    await web3Wrapper.awaitTransactionSuccessAsync(
+        await tokenTransferProxy.addAuthorizedAddress.sendTransactionAsync(exchange.address, { from: owner }),
+    );
+    await web3Wrapper.awaitTransactionSuccessAsync(
+        await tokenTransferProxy.transferOwnership.sendTransactionAsync(multiSig.address, { from: owner }),
+    );
     const addTokenGasEstimate = await tokenReg.addToken.estimateGasAsync(
         zrxToken.address,
         erc20TokenInfo[0].name,
@@ -80,29 +85,33 @@ export const runV1MigrationsAsync = async (provider: Provider, artifactsDir: str
         { from: owner },
     );
     const decimals = 18;
-    await tokenReg.addToken.sendTransactionAsync(
+    await web3Wrapper.awaitTransactionSuccessAsync(
-        zrxToken.address,
+        await tokenReg.addToken.sendTransactionAsync(
-        '0x Protocol Token',
+            zrxToken.address,
-        'ZRX',
+            '0x Protocol Token',
-        decimals,
+            'ZRX',
-        NULL_BYTES,
+            decimals,
-        NULL_BYTES,
+            NULL_BYTES,
-        {
+            NULL_BYTES,
-            from: owner,
+            {
-            gas: addTokenGasEstimate,
+                from: owner,
-        },
+                gas: addTokenGasEstimate,
+            },
+        ),
     );
-    await tokenReg.addToken.sendTransactionAsync(
+    await web3Wrapper.awaitTransactionSuccessAsync(
-        etherToken.address,
+        await tokenReg.addToken.sendTransactionAsync(
-        'Ether Token',
+            etherToken.address,
-        'WETH',
+            'Ether Token',
-        decimals,
+            'WETH',
-        NULL_BYTES,
+            decimals,
-        NULL_BYTES,
+            NULL_BYTES,
-        {
+            NULL_BYTES,
-            from: owner,
+            {
-            gas: addTokenGasEstimate,
+                from: owner,
-        },
+                gas: addTokenGasEstimate,
+            },
+        ),
     );
     for (const token of erc20TokenInfo) {
         const totalSupply = new BigNumber(100000000000000000000);
@@ -115,17 +124,19 @@ export const runV1MigrationsAsync = async (provider: Provider, artifactsDir: str
             token.decimals,
             totalSupply,
         );
-        await tokenReg.addToken.sendTransactionAsync(
+        await web3Wrapper.awaitTransactionSuccessAsync(
-            dummyToken.address,
+            await tokenReg.addToken.sendTransactionAsync(
-            token.name,
+                dummyToken.address,
-            token.symbol,
+                token.name,
-            token.decimals,
+                token.symbol,
-            token.ipfsHash,
+                token.decimals,
-            token.swarmHash,
+                token.ipfsHash,
-            {
+                token.swarmHash,
-                from: owner,
+                {
-                gas: addTokenGasEstimate,
+                    from: owner,
-            },
+                    gas: addTokenGasEstimate,
+                },
+            ),
         );
     }
 };

packages/migrations/src/v2/artifacts.ts

@@ -1,11 +1,11 @@
 import { ContractArtifact } from '@0xproject/sol-compiler';
 
+import * as AssetProxyOwner from '../../artifacts/2.0.0/AssetProxyOwner.json';
 import * as DummyERC20Token from '../../artifacts/2.0.0/DummyERC20Token.json';
 import * as DummyERC721Token from '../../artifacts/2.0.0/DummyERC721Token.json';
 import * as ERC20Proxy from '../../artifacts/2.0.0/ERC20Proxy.json';
 import * as ERC721Proxy from '../../artifacts/2.0.0/ERC721Proxy.json';
 import * as Exchange from '../../artifacts/2.0.0/Exchange.json';
-import * as MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddress from '../../artifacts/2.0.0/MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddress.json';
 import * as WETH9 from '../../artifacts/2.0.0/WETH9.json';
 import * as ZRX from '../../artifacts/2.0.0/ZRXToken.json';
 
@@ -13,7 +13,7 @@ export const artifacts = {
     ZRX: (ZRX as any) as ContractArtifact,
     DummyERC20Token: (DummyERC20Token as any) as ContractArtifact,
     DummyERC721Token: (DummyERC721Token as any) as ContractArtifact,
-    MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddress: (MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddress as any) as ContractArtifact,
+    AssetProxyOwner: (AssetProxyOwner as any) as ContractArtifact,
     Exchange: (Exchange as any) as ContractArtifact,
     WETH9: (WETH9 as any) as ContractArtifact,
     ERC20Proxy: (ERC20Proxy as any) as ContractArtifact,

packages/migrations/src/v2/migration.ts

@@ -6,12 +6,12 @@ import { ArtifactWriter } from '../artifact_writer';
 import { erc20TokenInfo, erc721TokenInfo } from '../utils/token_info';
 
 import { artifacts } from './artifacts';
+import { AssetProxyOwnerContract } from './contract_wrappers/asset_proxy_owner';
 import { DummyERC20TokenContract } from './contract_wrappers/dummy_e_r_c20_token';
 import { DummyERC721TokenContract } from './contract_wrappers/dummy_e_r_c721_token';
 import { ERC20ProxyContract } from './contract_wrappers/e_r_c20_proxy';
 import { ERC721ProxyContract } from './contract_wrappers/e_r_c721_proxy';
 import { ExchangeContract } from './contract_wrappers/exchange';
-import { MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddressContract } from './contract_wrappers/multi_sig_wallet_with_time_lock_except_remove_authorized_address';
 import { WETH9Contract } from './contract_wrappers/weth9';
 import { ZRXTokenContract } from './contract_wrappers/zrx_token';
 
@@ -62,34 +62,29 @@ export const runV2MigrationsAsync = async (provider: Provider, artifactsDir: str
     const secondsRequired = new BigNumber(0);
     const owner = accounts[0];
 
-    // TODO(leonid) use `AssetProxyOwner` after https://github.com/0xProject/0x-monorepo/pull/571 is merged
+    // AssetProxyOwner
-    // ERC20 Multisig
+    const assetProxyOwner = await AssetProxyOwnerContract.deployFrom0xArtifactAsync(
-    const multiSigERC20 = await MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddressContract.deployFrom0xArtifactAsync(
+        artifacts.AssetProxyOwner,
-        artifacts.MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddress,
         provider,
         txDefaults,
         owners,
+        [erc20proxy.address, erc721proxy.address],
         confirmationsRequired,
         secondsRequired,
-        erc20proxy.address,
     );
-    artifactsWriter.saveArtifact(multiSigERC20);
+    artifactsWriter.saveArtifact(assetProxyOwner);
-    await erc20proxy.addAuthorizedAddress.sendTransactionAsync(exchange.address, { from: owner });
+    await web3Wrapper.awaitTransactionSuccessAsync(
-    await erc20proxy.transferOwnership.sendTransactionAsync(multiSigERC20.address, { from: owner });
+        await erc20proxy.addAuthorizedAddress.sendTransactionAsync(exchange.address, { from: owner }),
-
+    );
-    // ERC721 Multisig
+    await web3Wrapper.awaitTransactionSuccessAsync(
-    const multiSigERC721 = await MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddressContract.deployFrom0xArtifactAsync(
+        await erc20proxy.transferOwnership.sendTransactionAsync(assetProxyOwner.address, { from: owner }),
-        artifacts.MultiSigWalletWithTimeLockExceptRemoveAuthorizedAddress,
+    );
-        provider,
+    await web3Wrapper.awaitTransactionSuccessAsync(
-        txDefaults,
+        await erc721proxy.addAuthorizedAddress.sendTransactionAsync(exchange.address, { from: owner }),
-        owners,
+    );
-        confirmationsRequired,
+    await web3Wrapper.awaitTransactionSuccessAsync(
-        secondsRequired,
+        await erc721proxy.transferOwnership.sendTransactionAsync(assetProxyOwner.address, { from: owner }),
-        erc721proxy.address,
     );
-    artifactsWriter.saveArtifact(multiSigERC721);
-    await erc721proxy.addAuthorizedAddress.sendTransactionAsync(exchange.address, { from: owner });
-    await erc721proxy.transferOwnership.sendTransactionAsync(multiSigERC721.address, { from: owner });
 
     // Dummy ERC20 tokens
     for (const token of erc20TokenInfo) {

packages/order-utils/CHANGELOG.json

@@ -2,7 +2,8 @@
     {
         "changes": [
             {
-                "note": "Export parseECSignature method"
+                "note": "Export parseECSignature method",
+                "pr": 684
             }
         ]
     },

packages/sol-cov/CHANGELOG.json

@@ -1,4 +1,45 @@
 [
+    {
+        "version": "0.2.0",
+        "changes": [
+            {
+                "note": "Fixed a bug causing RegExp to crash if contract code is longer that 32767 characters",
+                "pr": 675
+            },
+            {
+                "note": "Fixed a bug caused by Geth debug trace depth being 1-indexed",
+                "pr": 675
+            },
+            {
+                "note": "Fixed a bug when the tool crashed on empty traces",
+                "pr": 675
+            },
+            {
+                "note": "Use `BlockchainLifecycle` to support reverts on Geth",
+                "pr": 675
+            },
+            {
+                "note": "Add `ProfilerSubprovider` as a hacky way to profile code using coverage tools",
+                "pr": 675
+            },
+            {
+                "note": "Collect traces from `estimate_gas` calls",
+                "pr": 675
+            },
+            {
+                "note": "Fix a race condition caused by not awaiting the transaction before getting a trace",
+                "pr": 675
+            },
+            {
+                "note": "Add `start`/`stop` functionality to `CoverageSubprovider` and `ProfilerSubprovider`",
+                "pr": 675
+            },
+            {
+                "note": "Skip interface artifacts with a warning instead of failing",
+                "pr": 675
+            }
+        ]
+    },
     {
         "timestamp": 1527009134,
         "version": "0.1.0",

packages/sol-cov/package.json

@@ -54,6 +54,8 @@
         "@0xproject/subproviders": "^0.10.2",
         "@0xproject/typescript-typings": "^0.3.2",
         "@0xproject/utils": "^0.6.2",
+        "@0xproject/web3-wrapper": "^0.6.4",
+        "@0xproject/dev-utils": "^0.4.2",
         "ethereum-types": "^0.0.1",
         "ethereumjs-util": "^5.1.1",
         "glob": "^7.1.2",

packages/sol-cov/src/artifact_adapters/sol_compiler_artifact_adapter.ts

@@ -1,3 +1,5 @@
+import { ContractArtifact } from '@0xproject/sol-compiler';
+import { logUtils } from '@0xproject/utils';
 import * as fs from 'fs';
 import * as glob from 'glob';
 import * as _ from 'lodash';
@@ -18,18 +20,22 @@ export class SolCompilerArtifactAdapter extends AbstractArtifactAdapter {
         if (_.isUndefined(artifactsPath) && _.isUndefined(config.artifactsDir)) {
             throw new Error(`artifactsDir not found in ${CONFIG_FILE}`);
         }
-        this._artifactsPath = config.artifactsDir;
+        this._artifactsPath = artifactsPath || config.artifactsDir;
         if (_.isUndefined(sourcesPath) && _.isUndefined(config.contractsDir)) {
             throw new Error(`contractsDir not found in ${CONFIG_FILE}`);
         }
-        this._sourcesPath = config.contractsDir;
+        this._sourcesPath = sourcesPath || config.contractsDir;
     }
     public async collectContractsDataAsync(): Promise<ContractData[]> {
         const artifactsGlob = `${this._artifactsPath}/**/*.json`;
         const artifactFileNames = glob.sync(artifactsGlob, { absolute: true });
         const contractsData: ContractData[] = [];
         for (const artifactFileName of artifactFileNames) {
-            const artifact = JSON.parse(fs.readFileSync(artifactFileName).toString());
+            const artifact: ContractArtifact = JSON.parse(fs.readFileSync(artifactFileName).toString());
+            if (_.isUndefined(artifact.compilerOutput.evm)) {
+                logUtils.warn(`${artifactFileName} doesn't contain bytecode. Skipping...`);
+                continue;
+            }
             let sources = _.keys(artifact.sources);
             sources = _.map(sources, relativeFilePath => path.resolve(this._sourcesPath, relativeFilePath));
             const sourceCodes = _.map(sources, (source: string) => fs.readFileSync(source).toString());

packages/sol-cov/src/coverage_manager.ts

@@ -21,6 +21,7 @@ import {
     SourceRange,
     StatementCoverage,
     StatementDescription,
+    Subtrace,
     TraceInfo,
     TraceInfoExistingContract,
     TraceInfoNewContract,
@@ -29,21 +30,30 @@ import { utils } from './utils';
 
 const mkdirpAsync = promisify<undefined>(mkdirp);
 
+/**
+ * CoverageManager is used by CoverageSubprovider to compute code coverage based on collected trace data.
+ */
 export class CoverageManager {
     private _artifactAdapter: AbstractArtifactAdapter;
     private _logger: Logger;
     private _traceInfos: TraceInfo[] = [];
-    // tslint:disable-next-line:no-unused-variable
+    /**
-    private _getContractCodeAsync: (address: string) => Promise<string>;
+     * Computed partial coverage for a single file & subtrace
-    private static _getSingleFileCoverageForTrace(
+     * @param contractData      Contract metadata (source, srcMap, bytecode)
+     * @param subtrace          A subset of a transcation/call trace that was executed within that contract
+     * @param pcToSourceRange   A mapping from program counters to source ranges
+     * @param fileIndex         Index of a file to compute coverage for
+     * @return Partial istanbul coverage for that file & subtrace
+     */
+    private static _getSingleFileCoverageForSubtrace(
         contractData: ContractData,
-        coveredPcs: number[],
+        subtrace: Subtrace,
         pcToSourceRange: { [programCounter: number]: SourceRange },
         fileIndex: number,
     ): Coverage {
         const absoluteFileName = contractData.sources[fileIndex];
         const coverageEntriesDescription = collectCoverageEntries(contractData.sourceCodes[fileIndex]);
-        let sourceRanges = _.map(coveredPcs, coveredPc => pcToSourceRange[coveredPc]);
+        let sourceRanges = _.map(subtrace, structLog => pcToSourceRange[structLog.pc]);
         sourceRanges = _.compact(sourceRanges); // Some PC's don't map to a source range and we just ignore them.
         // By default lodash does a shallow object comparasion. We JSON.stringify them and compare as strings.
         sourceRanges = _.uniqBy(sourceRanges, s => JSON.stringify(s)); // We don't care if one PC was covered multiple times within a single transaction
@@ -52,26 +62,32 @@ export class CoverageManager {
         const branchIds = _.keys(coverageEntriesDescription.branchMap);
         for (const branchId of branchIds) {
             const branchDescription = coverageEntriesDescription.branchMap[branchId];
-            const isCoveredByBranchIndex = _.map(branchDescription.locations, location =>
+            const isBranchCoveredByBranchIndex = _.map(branchDescription.locations, location => {
-                _.some(sourceRanges, range => utils.isRangeInside(range.location, location)),
+                const isBranchCovered = _.some(sourceRanges, range => utils.isRangeInside(range.location, location));
-            );
+                const timesBranchCovered = Number(isBranchCovered);
-            branchCoverage[branchId] = isCoveredByBranchIndex;
+                return timesBranchCovered;
+            });
+            branchCoverage[branchId] = isBranchCoveredByBranchIndex;
         }
         const statementCoverage: StatementCoverage = {};
         const statementIds = _.keys(coverageEntriesDescription.statementMap);
         for (const statementId of statementIds) {
             const statementDescription = coverageEntriesDescription.statementMap[statementId];
-            const isCovered = _.some(sourceRanges, range => utils.isRangeInside(range.location, statementDescription));
+            const isStatementCovered = _.some(sourceRanges, range =>
-            statementCoverage[statementId] = isCovered;
+                utils.isRangeInside(range.location, statementDescription),
+            );
+            const timesStatementCovered = Number(isStatementCovered);
+            statementCoverage[statementId] = timesStatementCovered;
         }
         const functionCoverage: FunctionCoverage = {};
         const functionIds = _.keys(coverageEntriesDescription.fnMap);
         for (const fnId of functionIds) {
             const functionDescription = coverageEntriesDescription.fnMap[fnId];
-            const isCovered = _.some(sourceRanges, range =>
+            const isFunctionCovered = _.some(sourceRanges, range =>
                 utils.isRangeInside(range.location, functionDescription.loc),
             );
-            functionCoverage[fnId] = isCovered;
+            const timesFunctionCovered = Number(isFunctionCovered);
+            functionCoverage[fnId] = timesFunctionCovered;
         }
         // HACK: Solidity doesn't emit any opcodes that map back to modifiers with no args, that's why we map back to the
         // function range and check if there is any covered statement within that range.
@@ -95,12 +111,12 @@ export class CoverageManager {
                     return isInsideTheModifierEnclosingFunction && isCovered;
                 },
             );
-            statementCoverage[modifierStatementId] = isModifierCovered;
+            const timesModifierCovered = Number(isModifierCovered);
+            statementCoverage[modifierStatementId] = timesModifierCovered;
         }
         const partialCoverage = {
             [absoluteFileName]: {
                 ...coverageEntriesDescription,
-                l: {}, // It's able to derive it from statement coverage
                 path: absoluteFileName,
                 f: functionCoverage,
                 s: statementCoverage,
@@ -109,37 +125,7 @@ export class CoverageManager {
         };
         return partialCoverage;
     }
-    private static _bytecodeToBytecodeRegex(bytecode: string): string {
+    constructor(artifactAdapter: AbstractArtifactAdapter, isVerbose: boolean) {
-        const bytecodeRegex = bytecode
-            // Library linking placeholder: __ConvertLib____________________________
-            .replace(/_.*_/, '.*')
-            // Last 86 characters is solidity compiler metadata that's different between compilations
-            .replace(/.{86}$/, '')
-            // Libraries contain their own address at the beginning of the code and it's impossible to know it in advance
-            .replace(/^0x730000000000000000000000000000000000000000/, '0x73........................................');
-        return bytecodeRegex;
-    }
-    private static _getContractDataIfExists(contractsData: ContractData[], bytecode: string): ContractData | undefined {
-        if (!bytecode.startsWith('0x')) {
-            throw new Error(`0x hex prefix missing: ${bytecode}`);
-        }
-        const contractData = _.find(contractsData, contractDataCandidate => {
-            const bytecodeRegex = CoverageManager._bytecodeToBytecodeRegex(contractDataCandidate.bytecode);
-            const runtimeBytecodeRegex = CoverageManager._bytecodeToBytecodeRegex(
-                contractDataCandidate.runtimeBytecode,
-            );
-            // We use that function to find by bytecode or runtimeBytecode. Those are quasi-random strings so
-            // collisions are practically impossible and it allows us to reuse that code
-            return !_.isNull(bytecode.match(bytecodeRegex)) || !_.isNull(bytecode.match(runtimeBytecodeRegex));
-        });
-        return contractData;
-    }
-    constructor(
-        artifactAdapter: AbstractArtifactAdapter,
-        getContractCodeAsync: (address: string) => Promise<string>,
-        isVerbose: boolean,
-    ) {
-        this._getContractCodeAsync = getContractCodeAsync;
         this._artifactAdapter = artifactAdapter;
         this._logger = getLogger('sol-cov');
         this._logger.setLevel(isVerbose ? levels.TRACE : levels.ERROR);
@@ -157,56 +143,34 @@ export class CoverageManager {
         const contractsData = await this._artifactAdapter.collectContractsDataAsync();
         const collector = new Collector();
         for (const traceInfo of this._traceInfos) {
-            if (traceInfo.address !== constants.NEW_CONTRACT) {
+            const isContractCreation = traceInfo.address === constants.NEW_CONTRACT;
-                // Runtime transaction
+            const bytecode = isContractCreation
-                const runtimeBytecode = (traceInfo as TraceInfoExistingContract).runtimeBytecode;
+                ? (traceInfo as TraceInfoNewContract).bytecode
-                const contractData = CoverageManager._getContractDataIfExists(contractsData, runtimeBytecode);
+                : (traceInfo as TraceInfoExistingContract).runtimeBytecode;
-                if (_.isUndefined(contractData)) {
+            const contractData = utils.getContractDataIfExists(contractsData, bytecode);
-                    this._logger.warn(`Transaction to an unknown address: ${traceInfo.address}`);
+            if (_.isUndefined(contractData)) {
-                    continue;
+                const errMsg = isContractCreation
-                }
+                    ? `Unknown contract creation transaction`
-                const bytecodeHex = stripHexPrefix(runtimeBytecode);
+                    : `Transaction to an unknown address: ${traceInfo.address}`;
-                const sourceMap = contractData.sourceMapRuntime;
+                this._logger.warn(errMsg);
-                const pcToSourceRange = parseSourceMap(
+                continue;
-                    contractData.sourceCodes,
+            }
-                    sourceMap,
+            const bytecodeHex = stripHexPrefix(bytecode);
-                    bytecodeHex,
+            const sourceMap = isContractCreation ? contractData.sourceMap : contractData.sourceMapRuntime;
-                    contractData.sources,
+            const pcToSourceRange = parseSourceMap(
+                contractData.sourceCodes,
+                sourceMap,
+                bytecodeHex,
+                contractData.sources,
+            );
+            for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) {
+                const singleFileCoverageForTrace = CoverageManager._getSingleFileCoverageForSubtrace(
+                    contractData,
+                    traceInfo.subtrace,
+                    pcToSourceRange,
+                    fileIndex,
                 );
-                for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) {
+                collector.add(singleFileCoverageForTrace);
-                    const singleFileCoverageForTrace = CoverageManager._getSingleFileCoverageForTrace(
-                        contractData,
-                        traceInfo.coveredPcs,
-                        pcToSourceRange,
-                        fileIndex,
-                    );
-                    collector.add(singleFileCoverageForTrace);
-                }
-            } else {
-                // Contract creation transaction
-                const bytecode = (traceInfo as TraceInfoNewContract).bytecode;
-                const contractData = CoverageManager._getContractDataIfExists(contractsData, bytecode);
-                if (_.isUndefined(contractData)) {
-                    this._logger.warn(`Unknown contract creation transaction`);
-                    continue;
-                }
-                const bytecodeHex = stripHexPrefix(bytecode);
-                const sourceMap = contractData.sourceMap;
-                const pcToSourceRange = parseSourceMap(
-                    contractData.sourceCodes,
-                    sourceMap,
-                    bytecodeHex,
-                    contractData.sources,
-                );
-                for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) {
-                    const singleFileCoverageForTrace = CoverageManager._getSingleFileCoverageForTrace(
-                        contractData,
-                        traceInfo.coveredPcs,
-                        pcToSourceRange,
-                        fileIndex,
-                    );
-                    collector.add(singleFileCoverageForTrace);
-                }
             }
         }
         return collector.getFinalCoverage();

packages/sol-cov/src/coverage_subprovider.ts

@@ -1,31 +1,15 @@
-import { Callback, ErrorCallback, NextCallback, Subprovider } from '@0xproject/subproviders';
-import { BlockParam, CallData, JSONRPCRequestPayload, TransactionTrace, TxData } from 'ethereum-types';
 import * as _ from 'lodash';
-import { Lock } from 'semaphore-async-await';
 
 import { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter';
-import { constants } from './constants';
 import { CoverageManager } from './coverage_manager';
-import { getTracesByContractAddress } from './trace';
+import { TraceCollectionSubprovider } from './trace_collection_subprovider';
-import { BlockParamLiteral, TraceInfoExistingContract, TraceInfoNewContract } from './types';
-
-interface MaybeFakeTxData extends TxData {
-    isFakeTransaction?: boolean;
-}
-
-// Because there is no notion of a call trace in the Ethereum rpc - we collect them in a rather non-obvious/hacky way.
-// On each call - we create a snapshot, execute the call as a transaction, get the trace, revert the snapshot.
-// That allows us to avoid influencing test behaviour.
 
 /**
  * This class implements the [web3-provider-engine](https://github.com/MetaMask/provider-engine) subprovider interface.
- * It collects traces of all transactions that were sent and all calls that were executed through JSON RPC.
+ * It's used to compute your code coverage while running solidity tests.
  */
-export class CoverageSubprovider extends Subprovider {
+export class CoverageSubprovider extends TraceCollectionSubprovider {
-    // Lock is used to not accept normal transactions while doing call/snapshot magic because they'll be reverted later otherwise
-    private _lock: Lock;
     private _coverageManager: CoverageManager;
-    private _defaultFromAddress: string;
     /**
      * Instantiates a CoverageSubprovider instance
      * @param artifactAdapter Adapter for used artifacts format (0x, truffle, giveth, etc.)
@@ -33,172 +17,20 @@ export class CoverageSubprovider extends Subprovider {
      * @param isVerbose If true, we will log any unknown transactions. Otherwise we will ignore them
      */
     constructor(artifactAdapter: AbstractArtifactAdapter, defaultFromAddress: string, isVerbose: boolean = true) {
-        super();
+        const traceCollectionSubproviderConfig = {
-        this._lock = new Lock();
+            shouldCollectTransactionTraces: true,
-        this._defaultFromAddress = defaultFromAddress;
+            shouldCollectGasEstimateTraces: true,
-        this._coverageManager = new CoverageManager(artifactAdapter, this._getContractCodeAsync.bind(this), isVerbose);
+            shouldCollectCallTraces: true,
+        };
+        super(defaultFromAddress, traceCollectionSubproviderConfig);
+        this._coverageManager = new CoverageManager(artifactAdapter, isVerbose);
     }
     /**
      * Write the test coverage results to a file in Istanbul format.
      */
     public async writeCoverageAsync(): Promise<void> {
+        const traceInfos = this.getCollectedTraceInfos();
+        _.forEach(traceInfos, traceInfo => this._coverageManager.appendTraceInfo(traceInfo));
         await this._coverageManager.writeCoverageAsync();
     }
-    /**
-     * This method conforms to the web3-provider-engine interface.
-     * It is called internally by the ProviderEngine when it is this subproviders
-     * turn to handle a JSON RPC request.
-     * @param payload JSON RPC payload
-     * @param next Callback to call if this subprovider decides not to handle the request
-     * @param end Callback to call if subprovider handled the request and wants to pass back the request.
-     */
-    // tslint:disable-next-line:prefer-function-over-method async-suffix
-    public async handleRequest(payload: JSONRPCRequestPayload, next: NextCallback, end: ErrorCallback): Promise<void> {
-        switch (payload.method) {
-            case 'eth_sendTransaction':
-                const txData = payload.params[0];
-                next(this._onTransactionSentAsync.bind(this, txData));
-                return;
-
-            case 'eth_call':
-                const callData = payload.params[0];
-                const blockNumber = payload.params[1];
-                next(this._onCallExecutedAsync.bind(this, callData, blockNumber));
-                return;
-
-            default:
-                next();
-                return;
-        }
-    }
-    private async _onTransactionSentAsync(
-        txData: MaybeFakeTxData,
-        err: Error | null,
-        txHash: string | undefined,
-        cb: Callback,
-    ): Promise<void> {
-        if (!txData.isFakeTransaction) {
-            // This transaction is a usual ttransaction. Not a call executed as one.
-            // And we don't want it to be executed within a snapshotting period
-            await this._lock.acquire();
-        }
-        if (_.isNull(err)) {
-            const toAddress = _.isUndefined(txData.to) || txData.to === '0x0' ? constants.NEW_CONTRACT : txData.to;
-            await this._recordTxTraceAsync(toAddress, txData.data, txHash as string);
-        } else {
-            const payload = {
-                method: 'eth_getBlockByNumber',
-                params: [BlockParamLiteral.Latest, true],
-            };
-            const jsonRPCResponsePayload = await this.emitPayloadAsync(payload);
-            const transactions = jsonRPCResponsePayload.result.transactions;
-            for (const transaction of transactions) {
-                const toAddress = _.isUndefined(txData.to) || txData.to === '0x0' ? constants.NEW_CONTRACT : txData.to;
-                await this._recordTxTraceAsync(toAddress, transaction.data, transaction.hash);
-            }
-        }
-        if (!txData.isFakeTransaction) {
-            // This transaction is a usual ttransaction. Not a call executed as one.
-            // And we don't want it to be executed within a snapshotting period
-            this._lock.release();
-        }
-        cb();
-    }
-    private async _onCallExecutedAsync(
-        callData: Partial<CallData>,
-        blockNumber: BlockParam,
-        err: Error | null,
-        callResult: string,
-        cb: Callback,
-    ): Promise<void> {
-        await this._recordCallTraceAsync(callData, blockNumber);
-        cb();
-    }
-    private async _recordTxTraceAsync(address: string, data: string | undefined, txHash: string): Promise<void> {
-        let payload = {
-            method: 'debug_traceTransaction',
-            params: [txHash, { disableMemory: true, disableStack: false, disableStorage: true }],
-        };
-        let jsonRPCResponsePayload = await this.emitPayloadAsync(payload);
-        const trace: TransactionTrace = jsonRPCResponsePayload.result;
-        const tracesByContractAddress = getTracesByContractAddress(trace.structLogs, address);
-        const subcallAddresses = _.keys(tracesByContractAddress);
-        if (address === constants.NEW_CONTRACT) {
-            for (const subcallAddress of subcallAddresses) {
-                let traceInfo: TraceInfoNewContract | TraceInfoExistingContract;
-                if (subcallAddress === 'NEW_CONTRACT') {
-                    const traceForThatSubcall = tracesByContractAddress[subcallAddress];
-                    const coveredPcs = _.map(traceForThatSubcall, log => log.pc);
-                    traceInfo = {
-                        coveredPcs,
-                        txHash,
-                        address: constants.NEW_CONTRACT,
-                        bytecode: data as string,
-                    };
-                } else {
-                    payload = { method: 'eth_getCode', params: [subcallAddress, BlockParamLiteral.Latest] };
-                    jsonRPCResponsePayload = await this.emitPayloadAsync(payload);
-                    const runtimeBytecode = jsonRPCResponsePayload.result;
-                    const traceForThatSubcall = tracesByContractAddress[subcallAddress];
-                    const coveredPcs = _.map(traceForThatSubcall, log => log.pc);
-                    traceInfo = {
-                        coveredPcs,
-                        txHash,
-                        address: subcallAddress,
-                        runtimeBytecode,
-                    };
-                }
-                this._coverageManager.appendTraceInfo(traceInfo);
-            }
-        } else {
-            for (const subcallAddress of subcallAddresses) {
-                payload = { method: 'eth_getCode', params: [subcallAddress, BlockParamLiteral.Latest] };
-                jsonRPCResponsePayload = await this.emitPayloadAsync(payload);
-                const runtimeBytecode = jsonRPCResponsePayload.result;
-                const traceForThatSubcall = tracesByContractAddress[subcallAddress];
-                const coveredPcs = _.map(traceForThatSubcall, log => log.pc);
-                const traceInfo: TraceInfoExistingContract = {
-                    coveredPcs,
-                    txHash,
-                    address: subcallAddress,
-                    runtimeBytecode,
-                };
-                this._coverageManager.appendTraceInfo(traceInfo);
-            }
-        }
-    }
-    private async _recordCallTraceAsync(callData: Partial<CallData>, blockNumber: BlockParam): Promise<void> {
-        // We don't want other transactions to be exeucted during snashotting period, that's why we lock the
-        // transaction execution for all transactions except our fake ones.
-        await this._lock.acquire();
-        const snapshotId = Number((await this.emitPayloadAsync({ method: 'evm_snapshot' })).result);
-        const fakeTxData: MaybeFakeTxData = {
-            isFakeTransaction: true, // This transaction (and only it) is allowed to come through when the lock is locked
-            ...callData,
-            from: callData.from || this._defaultFromAddress,
-        };
-        try {
-            await this.emitPayloadAsync({
-                method: 'eth_sendTransaction',
-                params: [fakeTxData],
-            });
-        } catch (err) {
-            // Even if this transaction failed - we've already recorded it's trace.
-        }
-        const jsonRPCResponse = await this.emitPayloadAsync({ method: 'evm_revert', params: [snapshotId] });
-        this._lock.release();
-        const didRevert = jsonRPCResponse.result;
-        if (!didRevert) {
-            throw new Error('Failed to revert the snapshot');
-        }
-    }
-    private async _getContractCodeAsync(address: string): Promise<string> {
-        const payload = {
-            method: 'eth_getCode',
-            params: [address, BlockParamLiteral.Latest],
-        };
-        const jsonRPCResponsePayload = await this.emitPayloadAsync(payload);
-        const contractCode: string = jsonRPCResponsePayload.result;
-        return contractCode;
-    }
 }

packages/sol-cov/src/index.ts

@@ -1,4 +1,6 @@
 export { CoverageSubprovider } from './coverage_subprovider';
+// HACK: ProfilerSubprovider is a hacky way to do profiling using coverage tools. Not production ready
+export { ProfilerSubprovider } from './profiler_subprovider';
 export { SolCompilerArtifactAdapter } from './artifact_adapters/sol_compiler_artifact_adapter';
 export { TruffleArtifactAdapter } from './artifact_adapters/truffle_artifact_adapter';
 export { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter';

packages/sol-cov/src/profiler_manager.ts

@@ -0,0 +1,134 @@
+import { promisify } from '@0xproject/utils';
+import { stripHexPrefix } from 'ethereumjs-util';
+import * as fs from 'fs';
+import { Collector } from 'istanbul';
+import * as _ from 'lodash';
+import { getLogger, levels, Logger } from 'loglevel';
+import * as mkdirp from 'mkdirp';
+
+import { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter';
+import { collectCoverageEntries } from './collect_coverage_entries';
+import { constants } from './constants';
+import { parseSourceMap } from './source_maps';
+import {
+    ContractData,
+    Coverage,
+    SingleFileSourceRange,
+    SourceRange,
+    Subtrace,
+    TraceInfo,
+    TraceInfoExistingContract,
+    TraceInfoNewContract,
+} from './types';
+import { utils } from './utils';
+
+const mkdirpAsync = promisify<undefined>(mkdirp);
+
+/**
+ * ProfilerManager is used by ProfilerSubprovider to profile code while running Solidity tests based on collected trace data.
+ * HACK: It's almost the exact copy of CoverageManager but instead of reporting how much times was each statement executed - it reports - how expensive it was gaswise.
+ */
+export class ProfilerManager {
+    private _artifactAdapter: AbstractArtifactAdapter;
+    private _logger: Logger;
+    private _traceInfos: TraceInfo[] = [];
+    /**
+     * Computed partial coverage for a single file & subtrace
+     * @param contractData      Contract metadata (source, srcMap, bytecode)
+     * @param subtrace          A subset of a transcation/call trace that was executed within that contract
+     * @param pcToSourceRange   A mapping from program counters to source ranges
+     * @param fileIndex         Index of a file to compute coverage for
+     * @return Partial istanbul coverage for that file & subtrace
+     */
+    private static _getSingleFileCoverageForSubtrace(
+        contractData: ContractData,
+        subtrace: Subtrace,
+        pcToSourceRange: { [programCounter: number]: SourceRange },
+        fileIndex: number,
+    ): Coverage {
+        const absoluteFileName = contractData.sources[fileIndex];
+        const profilerEntriesDescription = collectCoverageEntries(contractData.sourceCodes[fileIndex]);
+        const gasConsumedByStatement: { [statementId: string]: number } = {};
+        const statementIds = _.keys(profilerEntriesDescription.statementMap);
+        for (const statementId of statementIds) {
+            const statementDescription = profilerEntriesDescription.statementMap[statementId];
+            const totalGasCost = _.sum(
+                _.map(subtrace, structLog => {
+                    const sourceRange = pcToSourceRange[structLog.pc];
+                    if (_.isUndefined(sourceRange)) {
+                        return 0;
+                    }
+                    if (sourceRange.fileName !== absoluteFileName) {
+                        return 0;
+                    }
+                    if (utils.isRangeInside(sourceRange.location, statementDescription)) {
+                        return structLog.gasCost;
+                    } else {
+                        return 0;
+                    }
+                }),
+            );
+            gasConsumedByStatement[statementId] = totalGasCost;
+        }
+        const partialProfilerOutput = {
+            [absoluteFileName]: {
+                ...profilerEntriesDescription,
+                path: absoluteFileName,
+                f: {}, // I's meaningless in profiling context
+                s: gasConsumedByStatement,
+                b: {}, // I's meaningless in profiling context
+            },
+        };
+        return partialProfilerOutput;
+    }
+    constructor(artifactAdapter: AbstractArtifactAdapter, isVerbose: boolean) {
+        this._artifactAdapter = artifactAdapter;
+        this._logger = getLogger('sol-cov');
+        this._logger.setLevel(isVerbose ? levels.TRACE : levels.ERROR);
+    }
+    public appendTraceInfo(traceInfo: TraceInfo): void {
+        this._traceInfos.push(traceInfo);
+    }
+    public async writeProfilerOutputAsync(): Promise<void> {
+        const finalCoverage = await this._computeCoverageAsync();
+        const stringifiedCoverage = JSON.stringify(finalCoverage, null, '\t');
+        await mkdirpAsync('coverage');
+        fs.writeFileSync('coverage/coverage.json', stringifiedCoverage);
+    }
+    private async _computeCoverageAsync(): Promise<Coverage> {
+        const contractsData = await this._artifactAdapter.collectContractsDataAsync();
+        const collector = new Collector();
+        for (const traceInfo of this._traceInfos) {
+            const isContractCreation = traceInfo.address === constants.NEW_CONTRACT;
+            const bytecode = isContractCreation
+                ? (traceInfo as TraceInfoNewContract).bytecode
+                : (traceInfo as TraceInfoExistingContract).runtimeBytecode;
+            const contractData = utils.getContractDataIfExists(contractsData, bytecode);
+            if (_.isUndefined(contractData)) {
+                const errMsg = isContractCreation
+                    ? `Unknown contract creation transaction`
+                    : `Transaction to an unknown address: ${traceInfo.address}`;
+                this._logger.warn(errMsg);
+                continue;
+            }
+            const bytecodeHex = stripHexPrefix(bytecode);
+            const sourceMap = isContractCreation ? contractData.sourceMap : contractData.sourceMapRuntime;
+            const pcToSourceRange = parseSourceMap(
+                contractData.sourceCodes,
+                sourceMap,
+                bytecodeHex,
+                contractData.sources,
+            );
+            for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) {
+                const singleFileCoverageForTrace = ProfilerManager._getSingleFileCoverageForSubtrace(
+                    contractData,
+                    traceInfo.subtrace,
+                    pcToSourceRange,
+                    fileIndex,
+                );
+                collector.add(singleFileCoverageForTrace);
+            }
+        }
+        return collector.getFinalCoverage();
+    }
+}

packages/sol-cov/src/profiler_subprovider.ts

@@ -0,0 +1,36 @@
+import * as _ from 'lodash';
+
+import { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter';
+import { ProfilerManager } from './profiler_manager';
+import { TraceCollectionSubprovider } from './trace_collection_subprovider';
+
+/**
+ * This class implements the [web3-provider-engine](https://github.com/MetaMask/provider-engine) subprovider interface.
+ * ProfilerSubprovider is used to profile Solidity code while running tests.
+ */
+export class ProfilerSubprovider extends TraceCollectionSubprovider {
+    private _profilerManager: ProfilerManager;
+    /**
+     * Instantiates a ProfilerSubprovider instance
+     * @param artifactAdapter Adapter for used artifacts format (0x, truffle, giveth, etc.)
+     * @param defaultFromAddress default from address to use when sending transactions
+     * @param isVerbose If true, we will log any unknown transactions. Otherwise we will ignore them
+     */
+    constructor(artifactAdapter: AbstractArtifactAdapter, defaultFromAddress: string, isVerbose: boolean = true) {
+        const traceCollectionSubproviderConfig = {
+            shouldCollectTransactionTraces: true,
+            shouldCollectGasEstimateTraces: false,
+            shouldCollectCallTraces: false,
+        };
+        super(defaultFromAddress, traceCollectionSubproviderConfig);
+        this._profilerManager = new ProfilerManager(artifactAdapter, isVerbose);
+    }
+    /**
+     * Write the test profiler results to a file in Istanbul format.
+     */
+    public async writeProfilerOutputAsync(): Promise<void> {
+        const traceInfos = this.getCollectedTraceInfos();
+        _.forEach(traceInfos, traceInfo => this._profilerManager.appendTraceInfo(traceInfo));
+        await this._profilerManager.writeProfilerOutputAsync();
+    }
+}

packages/sol-cov/src/trace.ts

@@ -16,6 +16,13 @@ export function getTracesByContractAddress(structLogs: StructLog[], startAddress
     const traceByContractAddress: TraceByContractAddress = {};
     let currentTraceSegment = [];
     const callStack = [startAddress];
+    if (_.isEmpty(structLogs)) {
+        return traceByContractAddress;
+    }
+    if (structLogs[0].depth === 1) {
+        // Geth uses 1-indexed depth counter whilst ganache starts from 0
+        _.forEach(structLogs, structLog => structLog.depth--);
+    }
     // tslint:disable-next-line:prefer-for-of
     for (let i = 0; i < structLogs.length; i++) {
         const structLog = structLogs[i];
@@ -95,10 +102,15 @@ export function getTracesByContractAddress(structLogs: StructLog[], startAddress
         }
     }
     if (callStack.length !== 0) {
-        throw new Error('Malformed trace. Call stack non empty at the end');
+        logUtils.warn('Malformed trace. Call stack non empty at the end');
     }
     if (currentTraceSegment.length !== 0) {
-        throw new Error('Malformed trace. Current trace segment non empty at the end');
+        const currentAddress = callStack.pop() as string;
+        traceByContractAddress[currentAddress] = (traceByContractAddress[currentAddress] || []).concat(
+            currentTraceSegment,
+        );
+        currentTraceSegment = [];
+        logUtils.warn('Malformed trace. Current trace segment non empty at the end');
     }
     return traceByContractAddress;
 }

packages/sol-cov/src/trace_collection_subprovider.ts

@@ -0,0 +1,233 @@
+import { BlockchainLifecycle } from '@0xproject/dev-utils';
+import { Callback, ErrorCallback, NextCallback, Subprovider } from '@0xproject/subproviders';
+import { Web3Wrapper } from '@0xproject/web3-wrapper';
+import { CallData, JSONRPCRequestPayload, Provider, TxData } from 'ethereum-types';
+import * as _ from 'lodash';
+import { Lock } from 'semaphore-async-await';
+
+import { constants } from './constants';
+import { getTracesByContractAddress } from './trace';
+import { BlockParamLiteral, TraceInfo, TraceInfoExistingContract, TraceInfoNewContract } from './types';
+
+interface MaybeFakeTxData extends TxData {
+    isFakeTransaction?: boolean;
+}
+
+const BLOCK_GAS_LIMIT = 6000000;
+
+export interface TraceCollectionSubproviderConfig {
+    shouldCollectTransactionTraces: boolean;
+    shouldCollectCallTraces: boolean;
+    shouldCollectGasEstimateTraces: boolean;
+}
+
+// Because there is no notion of a call trace in the Ethereum rpc - we collect them in a rather non-obvious/hacky way.
+// On each call - we create a snapshot, execute the call as a transaction, get the trace, revert the snapshot.
+// That allows us to avoid influencing test behaviour.
+
+/**
+ * This class implements the [web3-provider-engine](https://github.com/MetaMask/provider-engine) subprovider interface.
+ * It collects traces of all transactions that were sent and all calls that were executed through JSON RPC.
+ */
+export class TraceCollectionSubprovider extends Subprovider {
+    // Lock is used to not accept normal transactions while doing call/snapshot magic because they'll be reverted later otherwise
+    private _lock = new Lock();
+    private _defaultFromAddress: string;
+    private _web3Wrapper!: Web3Wrapper;
+    private _traceInfos: TraceInfo[] = [];
+    private _isEnabled = true;
+    private _config: TraceCollectionSubproviderConfig;
+    /**
+     * Instantiates a TraceCollectionSubprovider instance
+     * @param defaultFromAddress default from address to use when sending transactions
+     */
+    constructor(defaultFromAddress: string, config: TraceCollectionSubproviderConfig) {
+        super();
+        this._defaultFromAddress = defaultFromAddress;
+        this._config = config;
+    }
+    /**
+     * Returns all trace infos collected by the subprovider so far
+     */
+    public getCollectedTraceInfos(): TraceInfo[] {
+        return this._traceInfos;
+    }
+    /**
+     * Starts trace collection
+     */
+    public start(): void {
+        this._isEnabled = true;
+    }
+    /**
+     * Stops trace collection
+     */
+    public stop(): void {
+        this._isEnabled = false;
+    }
+    /**
+     * This method conforms to the web3-provider-engine interface.
+     * It is called internally by the ProviderEngine when it is this subproviders
+     * turn to handle a JSON RPC request.
+     * @param payload JSON RPC payload
+     * @param next Callback to call if this subprovider decides not to handle the request
+     * @param end Callback to call if subprovider handled the request and wants to pass back the request.
+     */
+    // tslint:disable-next-line:prefer-function-over-method async-suffix
+    public async handleRequest(payload: JSONRPCRequestPayload, next: NextCallback, end: ErrorCallback): Promise<void> {
+        if (this._isEnabled) {
+            switch (payload.method) {
+                case 'eth_sendTransaction':
+                    if (!this._config.shouldCollectTransactionTraces) {
+                        next();
+                    } else {
+                        const txData = payload.params[0];
+                        next(this._onTransactionSentAsync.bind(this, txData));
+                    }
+                    return;
+
+                case 'eth_call':
+                    if (!this._config.shouldCollectCallTraces) {
+                        next();
+                    } else {
+                        const callData = payload.params[0];
+                        next(this._onCallOrGasEstimateExecutedAsync.bind(this, callData));
+                    }
+                    return;
+
+                case 'eth_estimateGas':
+                    if (!this._config.shouldCollectGasEstimateTraces) {
+                        next();
+                    } else {
+                        const estimateGasData = payload.params[0];
+                        next(this._onCallOrGasEstimateExecutedAsync.bind(this, estimateGasData));
+                    }
+                    return;
+
+                default:
+                    next();
+                    return;
+            }
+        } else {
+            next();
+            return;
+        }
+    }
+    /**
+     * Set's the subprovider's engine to the ProviderEngine it is added to.
+     * This is only called within the ProviderEngine source code, do not call
+     * directly.
+     */
+    public setEngine(engine: Provider): void {
+        super.setEngine(engine);
+        this._web3Wrapper = new Web3Wrapper(engine);
+    }
+    private async _onTransactionSentAsync(
+        txData: MaybeFakeTxData,
+        err: Error | null,
+        txHash: string | undefined,
+        cb: Callback,
+    ): Promise<void> {
+        if (!txData.isFakeTransaction) {
+            // This transaction is a usual transaction. Not a call executed as one.
+            // And we don't want it to be executed within a snapshotting period
+            await this._lock.acquire();
+        }
+        const NULL_ADDRESS = '0x0';
+        if (_.isNull(err)) {
+            const toAddress =
+                _.isUndefined(txData.to) || txData.to === NULL_ADDRESS ? constants.NEW_CONTRACT : txData.to;
+            await this._recordTxTraceAsync(toAddress, txData.data, txHash as string);
+        } else {
+            const latestBlock = await this._web3Wrapper.getBlockWithTransactionDataAsync(BlockParamLiteral.Latest);
+            const transactions = latestBlock.transactions;
+            for (const transaction of transactions) {
+                const toAddress =
+                    _.isUndefined(txData.to) || txData.to === NULL_ADDRESS ? constants.NEW_CONTRACT : txData.to;
+                await this._recordTxTraceAsync(toAddress, transaction.input, transaction.hash);
+            }
+        }
+        if (!txData.isFakeTransaction) {
+            // This transaction is a usual transaction. Not a call executed as one.
+            // And we don't want it to be executed within a snapshotting period
+            this._lock.release();
+        }
+        cb();
+    }
+    private async _onCallOrGasEstimateExecutedAsync(
+        callData: Partial<CallData>,
+        err: Error | null,
+        callResult: string,
+        cb: Callback,
+    ): Promise<void> {
+        await this._recordCallOrGasEstimateTraceAsync(callData);
+        cb();
+    }
+    private async _recordTxTraceAsync(address: string, data: string | undefined, txHash: string): Promise<void> {
+        await this._web3Wrapper.awaitTransactionMinedAsync(txHash);
+        const trace = await this._web3Wrapper.getTransactionTraceAsync(txHash, {
+            disableMemory: true,
+            disableStack: false,
+            disableStorage: true,
+        });
+        const tracesByContractAddress = getTracesByContractAddress(trace.structLogs, address);
+        const subcallAddresses = _.keys(tracesByContractAddress);
+        if (address === constants.NEW_CONTRACT) {
+            for (const subcallAddress of subcallAddresses) {
+                let traceInfo: TraceInfoNewContract | TraceInfoExistingContract;
+                if (subcallAddress === 'NEW_CONTRACT') {
+                    const traceForThatSubcall = tracesByContractAddress[subcallAddress];
+                    traceInfo = {
+                        subtrace: traceForThatSubcall,
+                        txHash,
+                        address: subcallAddress,
+                        bytecode: data as string,
+                    };
+                } else {
+                    const runtimeBytecode = await this._web3Wrapper.getContractCodeAsync(subcallAddress);
+                    const traceForThatSubcall = tracesByContractAddress[subcallAddress];
+                    traceInfo = {
+                        subtrace: traceForThatSubcall,
+                        txHash,
+                        address: subcallAddress,
+                        runtimeBytecode,
+                    };
+                }
+                this._traceInfos.push(traceInfo);
+            }
+        } else {
+            for (const subcallAddress of subcallAddresses) {
+                const runtimeBytecode = await this._web3Wrapper.getContractCodeAsync(subcallAddress);
+                const traceForThatSubcall = tracesByContractAddress[subcallAddress];
+                const traceInfo: TraceInfoExistingContract = {
+                    subtrace: traceForThatSubcall,
+                    txHash,
+                    address: subcallAddress,
+                    runtimeBytecode,
+                };
+                this._traceInfos.push(traceInfo);
+            }
+        }
+    }
+    private async _recordCallOrGasEstimateTraceAsync(callData: Partial<CallData>): Promise<void> {
+        // We don't want other transactions to be exeucted during snashotting period, that's why we lock the
+        // transaction execution for all transactions except our fake ones.
+        await this._lock.acquire();
+        const blockchainLifecycle = new BlockchainLifecycle(this._web3Wrapper);
+        await blockchainLifecycle.startAsync();
+        const fakeTxData: MaybeFakeTxData = {
+            gas: BLOCK_GAS_LIMIT,
+            isFakeTransaction: true, // This transaction (and only it) is allowed to come through when the lock is locked
+            ...callData,
+            from: callData.from || this._defaultFromAddress,
+        };
+        try {
+            const txHash = await this._web3Wrapper.sendTransactionAsync(fakeTxData);
+            await this._web3Wrapper.awaitTransactionMinedAsync(txHash);
+        } catch (err) {
+            // Even if this transaction failed - we've already recorded it's trace.
+            _.noop();
+        }
+        await blockchainLifecycle.revertAsync();
+        this._lock.release();
+    }
+}

packages/sol-cov/src/types.ts

@@ -1,3 +1,5 @@
+import { StructLog } from 'ethereum-types';
+
 export interface LineColumn {
     line: number;
     column: number;
@@ -45,24 +47,24 @@ export interface StatementMap {
 }
 
 export interface LineCoverage {
-    [lineNo: number]: boolean;
+    [lineNo: number]: number;
 }
 
 export interface FunctionCoverage {
-    [functionId: string]: boolean;
+    [functionId: string]: number;
 }
 
 export interface StatementCoverage {
-    [statementId: string]: boolean;
+    [statementId: string]: number;
 }
 
 export interface BranchCoverage {
-    [branchId: string]: boolean[];
+    [branchId: string]: number[];
 }
 
 export interface Coverage {
     [fineName: string]: {
-        l: LineCoverage;
+        l?: LineCoverage;
         f: FunctionCoverage;
         s: StatementCoverage;
         b: BranchCoverage;
@@ -82,8 +84,11 @@ export interface ContractData {
     sources: string[];
 }
 
+// Part of the trace executed within the same context
+export type Subtrace = StructLog[];
+
 export interface TraceInfoBase {
-    coveredPcs: number[];
+    subtrace: Subtrace;
     txHash: string;
 }
 

packages/sol-cov/src/utils.ts

@@ -1,4 +1,6 @@
-import { LineColumn, SingleFileSourceRange } from './types';
+import * as _ from 'lodash';
+
+import { ContractData, LineColumn, SingleFileSourceRange } from './types';
 
 export const utils = {
     compareLineColumn(lhs: LineColumn, rhs: LineColumn): number {
@@ -14,4 +16,30 @@ export const utils = {
             utils.compareLineColumn(childRange.end, parentRange.end) <= 0
         );
     },
+    bytecodeToBytecodeRegex(bytecode: string): string {
+        const bytecodeRegex = bytecode
+            // Library linking placeholder: __ConvertLib____________________________
+            .replace(/_.*_/, '.*')
+            // Last 86 characters is solidity compiler metadata that's different between compilations
+            .replace(/.{86}$/, '')
+            // Libraries contain their own address at the beginning of the code and it's impossible to know it in advance
+            .replace(/^0x730000000000000000000000000000000000000000/, '0x73........................................');
+        // HACK: Node regexes can't be longer that 32767 characters. Contracts bytecode can. We just truncate the regexes. It's safe in practice.
+        const MAX_REGEX_LENGTH = 32767;
+        const truncatedBytecodeRegex = bytecodeRegex.slice(0, MAX_REGEX_LENGTH);
+        return truncatedBytecodeRegex;
+    },
+    getContractDataIfExists(contractsData: ContractData[], bytecode: string): ContractData | undefined {
+        if (!bytecode.startsWith('0x')) {
+            throw new Error(`0x hex prefix missing: ${bytecode}`);
+        }
+        const contractData = _.find(contractsData, contractDataCandidate => {
+            const bytecodeRegex = utils.bytecodeToBytecodeRegex(contractDataCandidate.bytecode);
+            const runtimeBytecodeRegex = utils.bytecodeToBytecodeRegex(contractDataCandidate.runtimeBytecode);
+            // We use that function to find by bytecode or runtimeBytecode. Those are quasi-random strings so
+            // collisions are practically impossible and it allows us to reuse that code
+            return !_.isNull(bytecode.match(bytecodeRegex)) || !_.isNull(bytecode.match(runtimeBytecodeRegex));
+        });
+        return contractData;
+    },
 };

packages/subproviders/CHANGELOG.json

@@ -1,4 +1,13 @@
 [
+    {
+        "version": "0.10.3",
+        "changes": [
+            {
+                "note": "Define engine type as Provider in setEngine",
+                "pr": 675
+            }
+        ]
+    },
     {
         "timestamp": 1527009133,
         "version": "0.10.2",

packages/subproviders/src/subproviders/subprovider.ts

@@ -1,5 +1,5 @@
 import { promisify } from '@0xproject/utils';
-import { JSONRPCRequestPayload, JSONRPCResponsePayload } from 'ethereum-types';
+import { JSONRPCRequestPayload, JSONRPCResponsePayload, Provider } from 'ethereum-types';
 
 import { Callback, ErrorCallback, JSONRPCRequestPayloadWithMethod } from '../types';
 /**
@@ -8,7 +8,7 @@ import { Callback, ErrorCallback, JSONRPCRequestPayloadWithMethod } from '../typ
  */
 export abstract class Subprovider {
     // tslint:disable-next-line:underscore-private-and-protected
-    private engine: any;
+    private engine!: Provider;
     // Ported from: https://github.com/MetaMask/provider-engine/blob/master/util/random-id.js
     private static _getRandomId(): number {
         const extraDigits = 3;
@@ -56,7 +56,7 @@ export abstract class Subprovider {
      * This is only called within the ProviderEngine source code, do not call
      * directly.
      */
-    public setEngine(engine: any): void {
+    public setEngine(engine: Provider): void {
         this.engine = engine;
     }
 }

packages/web3-wrapper/CHANGELOG.json

@@ -2,6 +2,18 @@
     {
         "version": "0.7.0",
         "changes": [
+            {
+                "note": "Add `web3Wrapper.getContractCodeAsync`",
+                "pr": 675
+            },
+            {
+                "note": "Add `web3Wrapper.getTransactionTraceAsync`",
+                "pr": 675
+            },
+            {
+                "note": "Add `web3Wrapper.getBlockWithTransactionDataAsync`",
+                "pr": 675
+            },
             {
                 "note": "Add exported uniqueVersionIds object",
                 "pr": 622

packages/web3-wrapper/src/web3_wrapper.ts

@@ -2,6 +2,7 @@ import { AbiDecoder, addressUtils, BigNumber, intervalUtils, promisify } from '@
 import {
     BlockParam,
     BlockWithoutTransactionData,
+    BlockWithTransactionData,
     CallData,
     ContractAbi,
     FilterObject,
@@ -10,8 +11,10 @@ import {
     LogEntry,
     Provider,
     RawLogEntry,
+    TraceParams,
     TransactionReceipt,
     TransactionReceiptWithDecodedLogs,
+    TransactionTrace,
     TxData,
 } from 'ethereum-types';
 import * as _ from 'lodash';
@@ -187,11 +190,33 @@ export class Web3Wrapper {
      * @returns Whether or not contract code was found at the supplied address
      */
     public async doesContractExistAtAddressAsync(address: string): Promise<boolean> {
-        const code = await promisify<string>(this._web3.eth.getCode)(address);
+        const code = await this.getContractCodeAsync(address);
         // Regex matches 0x0, 0x00, 0x in order to accommodate poorly implemented clients
         const isCodeEmpty = /^0x0{0,40}$/i.test(code);
         return !isCodeEmpty;
     }
+    /**
+     * Gets the contract code by address
+     * @param  address Address of the contract
+     * @return Code of the contract
+     */
+    public async getContractCodeAsync(address: string): Promise<string> {
+        const code = await promisify<string>(this._web3.eth.getCode)(address);
+        return code;
+    }
+    /**
+     * Gets the debug trace of a transaction
+     * @param  txHash Hash of the transactuon to get a trace for
+     * @param  traceParams Config object allowing you to specify if you need memory/storage/stack traces.
+     * @return Transaction trace
+     */
+    public async getTransactionTraceAsync(txHash: string, traceParams: TraceParams): Promise<TransactionTrace> {
+        const trace = await this._sendRawPayloadAsync<TransactionTrace>({
+            method: 'debug_traceTransaction',
+            params: [txHash, traceParams],
+        });
+        return trace;
+    }
     /**
      * Sign a message with a specific address's private key (`eth_sign`)
      * @param address Address of signer
@@ -211,13 +236,30 @@ export class Web3Wrapper {
         return blockNumber;
     }
     /**
-     * Fetch a specific Ethereum block
+     * Fetch a specific Ethereum block without transaction data
      * @param blockParam The block you wish to fetch (blockHash, blockNumber or blockLiteral)
      * @returns The requested block without transaction data
      */
     public async getBlockAsync(blockParam: string | BlockParam): Promise<BlockWithoutTransactionData> {
-        const block = await promisify<BlockWithoutTransactionData>(this._web3.eth.getBlock)(blockParam);
+        const shouldIncludeTransactionData = false;
-        return block;
+        const blockWithoutTransactionData = await promisify<BlockWithoutTransactionData>(this._web3.eth.getBlock)(
+            blockParam,
+            shouldIncludeTransactionData,
+        );
+        return blockWithoutTransactionData;
+    }
+    /**
+     * Fetch a specific Ethereum block with transaction data
+     * @param blockParam The block you wish to fetch (blockHash, blockNumber or blockLiteral)
+     * @returns The requested block with transaction data
+     */
+    public async getBlockWithTransactionDataAsync(blockParam: string | BlockParam): Promise<BlockWithTransactionData> {
+        const shouldIncludeTransactionData = true;
+        const blockWithTransactionData = await promisify<BlockWithTransactionData>(this._web3.eth.getBlock)(
+            blockParam,
+            shouldIncludeTransactionData,
+        );
+        return blockWithTransactionData;
     }
     /**
      * Fetch a block's timestamp
@@ -469,4 +511,4 @@ export class Web3Wrapper {
         const decimal = this._web3.toDecimal(hex);
         return decimal;
     }
-}
+} // tslint:disable-line:max-file-line-count