crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement docker as another backend for sol-compiler

Browse Source
This commit is contained in:
Leonid Logvinov 2019-01-23 16:54:27 +01:00
parent ef928aff31
commit e8d68dc07f
No known key found for this signature in database
GPG Key ID: 0DD294BFDE8C95D4
14 changed files with 179 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contracts/examples/compiler.json
View File

@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
"workspaceDir": "../..",
"useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,

2
contracts/extensions/compiler.json
View File

@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
"workspaceDir": "../..",
"useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,

2
contracts/interfaces/compiler.json
View File

@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
"workspaceDir": "../..",
"useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,

2
contracts/libs/compiler.json
View File

@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
"workspaceDir": "../..",
"useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,

2
contracts/multisig/compiler.json
View File

@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
"workspaceDir": "../..",
"useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,

2
contracts/protocol/compiler.json
View File

@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
"workspaceDir": "../..",
"useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,

2
contracts/tokens/compiler.json
View File

@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
"workspaceDir": "../..",
"useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,

2
contracts/utils/compiler.json
View File

@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
"workspaceDir": "../..",
"useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,

6
packages/ethereum-types/src/index.ts
View File

@ -503,19 +503,23 @@ export interface Source {
/**
* Options you can specify (as flags or in a compiler.json file) when invoking sol-compiler
* contractsDir: Directory containing your project's Solidity contracts. Can contain nested directories.
* contractsDir: Directory containing your package's Solidity contracts. Can contain nested directories.
* workspaceDir: Directory containing your project's Solidity contracts. All the contracts used in compilation must be withing it. Similar to --allow-paths in Solidity.
* artifactsDir: Directory where you want the generated artifacts.json written to
* compilerSettings: Desired settings to pass to the Solidity compiler during compilation.
* (http://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#compiler-input-and-output-json-description)
* contracts: List of contract names you wish to compile, or alternatively ['*'] to compile all contracts in the
* specified directory.
* useDockerisedSolc: If set to true - sol-compiler will try using docker to achieve faster compilation times. Otherwise and by default - solcjs will be used.
* solcVersion: If you don't want to compile each contract with the Solidity version specified in-file, you can force all
* contracts to compile with the the version specified here.
*/
export interface CompilerOptions {
contractsDir?: string;
workspaceDir?: string;
artifactsDir?: string;
compilerSettings?: CompilerSettings;
contracts?: string[] | '*';
useDockerisedSolc?: boolean;
solcVersion?: string;
} // tslint:disable-line:max-file-line-count

76
packages/sol-compiler/src/compiler.ts
View File

@ -10,6 +10,7 @@ import {
URLResolver,
} from '@0x/sol-resolver';
import { logUtils } from '@0x/utils';
import { execSync } from 'child_process';
import * as chokidar from 'chokidar';
import { CompilerOptions, ContractArtifact, ContractVersionData, StandardOutput } from 'ethereum-types';
import * as fs from 'fs';
@ -23,10 +24,11 @@ import { compilerOptionsSchema } from './schemas/compiler_options_schema';
import { binPaths } from './solc/bin_paths';
import {
addHexPrefixToContractBytecode,
compile,
compileDocker,
compileSolcJS,
createDirIfDoesNotExistAsync,
getContractArtifactIfExistsAsync,
getSolcAsync,
getSolcJSAsync,
getSourcesWithDependencies,
getSourceTreeHash,
parseSolidityVersionRange,
@ -40,6 +42,7 @@ const ALL_CONTRACTS_IDENTIFIER = '*';
const ALL_FILES_IDENTIFIER = '*';
const DEFAULT_CONTRACTS_DIR = path.resolve('contracts');
const DEFAULT_ARTIFACTS_DIR = path.resolve('artifacts');
const DEFAULT_USE_DOCKERISED_SOLC = false;
// Solc compiler settings cannot be configured from the commandline.
// If you need this configured, please create a `compiler.json` config file
// with your desired configurations.
@ -80,10 +83,12 @@ export class Compiler {
private readonly _resolver: Resolver;
private readonly _nameResolver: NameResolver;
private readonly _contractsDir: string;
private readonly _workspaceDir: string;
private readonly _compilerSettings: solc.CompilerSettings;
private readonly _artifactsDir: string;
private readonly _solcVersionIfExists: string | undefined;
private readonly _specifiedContracts: string[] | TYPE_ALL_FILES_IDENTIFIER;
private readonly _useDockerisedSolc: boolean;
/**
* Instantiates a new instance of the Compiler class.
* @param opts Optional compiler options
@ -97,16 +102,23 @@ export class Compiler {
: {};
const passedOpts = opts || {};
assert.doesConformToSchema('compiler.json', config, compilerOptionsSchema);
this._contractsDir = passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR;
this._contractsDir = path.resolve(passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR);
this._workspaceDir = path.resolve(passedOpts.workspaceDir || config.workspaceDir || this._contractsDir);
if (!this._contractsDir.includes(this._workspaceDir)) {
throw new Error(
`Contracts dir ${this._contractsDir} is outside of the workspace dir ${this._workspaceDir}`,
);
}
this._solcVersionIfExists = passedOpts.solcVersion || config.solcVersion;
this._compilerSettings = passedOpts.compilerSettings || config.compilerSettings || DEFAULT_COMPILER_SETTINGS;
this._artifactsDir = passedOpts.artifactsDir || config.artifactsDir || DEFAULT_ARTIFACTS_DIR;
this._specifiedContracts = passedOpts.contracts || config.contracts || ALL_CONTRACTS_IDENTIFIER;
this._nameResolver = new NameResolver(path.resolve(this._contractsDir));
this._useDockerisedSolc =
passedOpts.useDockerisedSolc || config.useDockerisedSolc || DEFAULT_USE_DOCKERISED_SOLC;
this._nameResolver = new NameResolver(this._contractsDir);
const resolver = new FallthroughResolver();
resolver.appendResolver(new URLResolver());
const packagePath = path.resolve('');
resolver.appendResolver(new NPMResolver(packagePath));
resolver.appendResolver(new NPMResolver(this._contractsDir, this._workspaceDir));
resolver.appendResolver(new RelativeFSResolver(this._contractsDir));
resolver.appendResolver(new FSResolver());
resolver.appendResolver(this._nameResolver);
@ -205,11 +217,12 @@ export class Compiler {
// map contract paths to data about them for later verification and persistence
const contractPathToData: ContractPathToData = {};
const spyResolver = new SpyResolver(this._resolver);
for (const contractName of contractNames) {
const contractSource = this._resolver.resolve(contractName);
const contractSource = spyResolver.resolve(contractName);
const sourceTreeHashHex = getSourceTreeHash(
this._resolver,
spyResolver,
path.join(this._contractsDir, contractSource.path),
).toString('hex');
const contractData = {
@ -242,6 +255,30 @@ export class Compiler {
versionToInputs[solcVersion].contractsToCompile.push(contractSource.path);
}
const allTouchedFiles = spyResolver.resolvedContractSources.map(
contractSource => `${contractSource.absolutePath}`,
);
const NODE_MODULES = 'node_modules';
const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES));
const dependencyNameToPackagePath: { [dependencyName: string]: string } = {};
_.map(allTouchedDependencies, dependencyFilePath => {
const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES);
const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length;
const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1);
let packageName;
let packageScopeIfExists;
let dependencyName;
if (_.startsWith(importPath, '@')) {
[packageScopeIfExists, packageName] = importPath.split('/');
dependencyName = `${packageScopeIfExists}/${packageName}`;
} else {
[packageName] = importPath.split('/');
dependencyName = `${packageName}`;
}
const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName);
dependencyNameToPackagePath[dependencyName] = dependencyPackagePath;
});
const compilerOutputs: StandardOutput[] = [];
const solcVersions = _.keys(versionToInputs);
@ -252,12 +289,31 @@ export class Compiler {
input.contractsToCompile
}) with Solidity v${solcVersion}...`,
);
let compilerOutput;
let fullSolcVersion;
if (this._useDockerisedSolc) {
const dockerCommand = `docker run ethereum/solc:${solcVersion} --version`;
const versionCommandOutput = execSync(dockerCommand).toString();
const versionCommandOutputParts = versionCommandOutput.split(' ');
fullSolcVersion = versionCommandOutputParts[versionCommandOutputParts.length - 1].trim();
compilerOutput = compileDocker(
this._resolver,
this._contractsDir,
this._workspaceDir,
solcVersion,
dependencyNameToPackagePath,
input.standardInput,
);
} else {
fullSolcVersion = binPaths[solcVersion];
const solcInstance = await getSolcJSAsync(solcVersion);
compilerOutput = compileSolcJS(this._resolver, solcInstance, input.standardInput);
}
const { solcInstance, fullSolcVersion } = await getSolcAsync(solcVersion);
const compilerOutput = compile(this._resolver, solcInstance, input.standardInput);
compilerOutputs.push(compilerOutput);
for (const contractPath of input.contractsToCompile) {
// console.log('contractsPath', contractPath);
const contractName = contractPathToData[contractPath].contractName;
const compiledContract = compilerOutput.contracts[contractPath][contractName];

2
packages/sol-compiler/src/schemas/compiler_options_schema.ts
View File

@ -2,6 +2,7 @@ export const compilerOptionsSchema = {
id: '/CompilerOptions',
properties: {
contractsDir: { type: 'string' },
workspaceDir: { type: 'string' },
artifactsDir: { type: 'string' },
solcVersion: { type: 'string', pattern: '^\\d+.\\d+.\\d+$' },
compilerSettings: { type: 'object' },
@ -19,6 +20,7 @@ export const compilerOptionsSchema = {
},
],
},
useDockerisedSolc: { type: 'boolean' },
},
type: 'object',
required: [],

85
packages/sol-compiler/src/utils/compiler.ts
View File

@ -1,6 +1,7 @@
import { ContractSource, Resolver } from '@0x/sol-resolver';
import { fetchAsync, logUtils } from '@0x/utils';
import chalk from 'chalk';
import { execSync } from 'child_process';
import { ContractArtifact } from 'ethereum-types';
import * as ethUtil from 'ethereumjs-util';
import * as _ from 'lodash';
@ -121,7 +122,7 @@ export function parseDependencies(contractSource: ContractSource): string[] {
* @param solcInstance Instance of a solc compiler
* @param standardInput Solidity standard JSON input
*/
export function compile(
export function compileSolcJS(
resolver: Resolver,
solcInstance: solc.SolcInstance,
standardInput: solc.StandardInput,
@ -137,6 +138,80 @@ export function compile(
}
return compiled;
}
/**
* Compiles the contracts and prints errors/warnings
* @param resolver Resolver
* @param contractsDir Contracts directory
* @param workspaceDir Workspace directory
* @param solcVersion Version of a solc compiler
* @param dependencyNameToPackagePath Mapping of dependency name to it's package path
* @param standardInput Solidity standard JSON input
*/
export function compileDocker(
resolver: Resolver,
contractsDir: string,
workspaceDir: string,
solcVersion: string,
dependencyNameToPackagePath: { [dependencyName: string]: string },
standardInput: solc.StandardInput,
): solc.StandardOutput {
const standardInputDocker = _.cloneDeep(standardInput);
standardInputDocker.settings.remappings = _.map(
dependencyNameToPackagePath,
(dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`,
);
standardInputDocker.sources = _.mapKeys(
standardInputDocker.sources,
(_source: solc.Source, sourcePath: string) => resolver.resolve(sourcePath).absolutePath,
);
const standardInputStrDocker = JSON.stringify(standardInputDocker, null, 2);
const dockerCommand =
`docker run -i -a stdin -a stdout -a stderr -v ${workspaceDir}:${workspaceDir} ethereum/solc:${solcVersion} ` +
`solc --standard-json --allow-paths ${workspaceDir}`;
const standardOutputStrDocker = execSync(dockerCommand, { input: standardInputStrDocker }).toString();
const compiledDocker: solc.StandardOutput = JSON.parse(standardOutputStrDocker);
if (!_.isUndefined(compiledDocker.errors)) {
printCompilationErrorsAndWarnings(compiledDocker.errors);
}
compiledDocker.sources = makeContractPathsRelative(
compiledDocker.sources,
contractsDir,
dependencyNameToPackagePath,
);
compiledDocker.contracts = makeContractPathsRelative(
compiledDocker.contracts,
contractsDir,
dependencyNameToPackagePath,
);
return compiledDocker;
}
function makeContractPathRelative(
absolutePath: string,
contractsDir: string,
dependencyNameToPackagePath: { [dependencyName: string]: string },
): string {
let contractPath = absolutePath.replace(`${contractsDir}/`, '');
_.map(dependencyNameToPackagePath, (packagePath: string, dependencyName: string) => {
contractPath = contractPath.replace(packagePath, dependencyName);
});
return contractPath;
}
function makeContractPathsRelative(
absolutePathToSmth: { [absoluteContractPath: string]: any },
contractsDir: string,
dependencyNameToPackagePath: { [dependencyName: string]: string },
): { [contractPath: string]: any } {
return _.mapKeys(absolutePathToSmth, (_val: any, absoluteContractPath: string) =>
makeContractPathRelative(absoluteContractPath, contractsDir, dependencyNameToPackagePath),
);
}
/**
* Separates errors from warnings, formats the messages and prints them. Throws if there is any compilation error (not warning).
* @param solcErrors The errors field of standard JSON output that contains errors and warnings.
@ -267,13 +342,11 @@ function recursivelyGatherDependencySources(
}
/**
* Gets the solidity compiler instance and full version name. If the compiler is already cached - gets it from FS,
* Gets the solidity compiler instance. If the compiler is already cached - gets it from FS,
* otherwise - fetches it and caches it.
* @param solcVersion The compiler version. e.g. 0.5.0
*/
export async function getSolcAsync(
solcVersion: string,
): Promise<{ solcInstance: solc.SolcInstance; fullSolcVersion: string }> {
export async function getSolcJSAsync(solcVersion: string): Promise<solc.SolcInstance> {
const fullSolcVersion = binPaths[solcVersion];
if (_.isUndefined(fullSolcVersion)) {
throw new Error(`${solcVersion} is not a known compiler version`);
@ -297,7 +370,7 @@ export async function getSolcAsync(
throw new Error('No compiler available');
}
const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
return { solcInstance, fullSolcVersion };
return solcInstance;
}
/**

4
packages/sol-resolver/src/resolvers/fs_resolver.ts
View File

@ -1,4 +1,5 @@
import * as fs from 'fs';
import * as path from 'path';
import { ContractSource } from '../types';
@ -9,7 +10,8 @@ export class FSResolver extends Resolver {
public resolveIfExists(importPath: string): ContractSource | undefined {
if (fs.existsSync(importPath) && fs.lstatSync(importPath).isFile()) {
const fileContent = fs.readFileSync(importPath).toString();
return { source: fileContent, path: importPath, absolutePath: importPath };
const absolutePath = path.resolve(importPath);
return { source: fileContent, path: importPath, absolutePath } as any;
}
return undefined;
}

13
packages/sol-resolver/src/resolvers/npm_resolver.ts
View File

@ -8,9 +8,11 @@ import { Resolver } from './resolver';
export class NPMResolver extends Resolver {
private readonly _packagePath: string;
constructor(packagePath: string) {
private readonly _workspacePath: string;
constructor(packagePath: string, workspacePath: string) {
super();
this._packagePath = packagePath;
this._workspacePath = workspacePath;
}
public resolveIfExists(importPath: string): ContractSource | undefined {
if (!importPath.startsWith('/')) {
@ -23,9 +25,11 @@ export class NPMResolver extends Resolver {
[packageName, ...other] = importPath.split('/');
}
const pathWithinPackage = path.join(...other);
let currentPath = this._packagePath;
const ROOT_PATH = '/';
while (currentPath !== ROOT_PATH) {
for (
let currentPath = this._packagePath;
currentPath.includes(this._workspacePath);
currentPath = path.dirname(currentPath)
) {
const packagePath = _.isUndefined(packageScopeIfExists)
? packageName
: path.join(packageScopeIfExists, packageName);
@ -34,7 +38,6 @@ export class NPMResolver extends Resolver {
const fileContent = fs.readFileSync(lookupPath).toString();
return { source: fileContent, path: importPath, absolutePath: lookupPath };
}
currentPath = path.dirname(currentPath);
}
}
return undefined;