Merge pull request #1439 from 0xProject/migrations/docker-image

Add Docker image and Snapshot commands for migrations

.circleci/config.yml

@@ -188,9 +188,7 @@ jobs:
         working_directory: ~/repo
         docker:
             - image: circleci/python
-            - image: 0xorg/ganache-cli
+            - image: 0xorg/ganache-cli:2.2.2
-              command: |
-                  ganache-cli --gasLimit 10000000 --noVMErrorsOnRPCResponse --db /snapshot --noVMErrorsOnRPCResponse -p 8545 --networkId 50 -m "concert load couple harbor equip island argue ramp clarify fence smart topic"
             - image: 0xorg/launch-kit-ci
               command: |
                   yarn start:ts -p 3000:3000

packages/dev-utils/src/web3_factory.ts

@@ -17,6 +17,7 @@ export interface Web3Config {
     shouldThrowErrorsOnGanacheRPCResponse?: boolean; // default: true
     rpcUrl?: string; // default: localhost:8545
     shouldUseFakeGasEstimate?: boolean; // default: true
+    ganacheDatabasePath?: string; // default: undefined, creates a tmp dir
 }
 
 export const web3Factory = {
@@ -45,9 +46,14 @@ export const web3Factory = {
             const shouldThrowErrorsOnGanacheRPCResponse =
                 _.isUndefined(config.shouldThrowErrorsOnGanacheRPCResponse) ||
                 config.shouldThrowErrorsOnGanacheRPCResponse;
+            if (!_.isUndefined(config.ganacheDatabasePath)) {
+                // Saving the snapshot to a local db. Ganache requires this directory to exist
+                fs.mkdirSync(config.ganacheDatabasePath);
+            }
             provider.addProvider(
                 new GanacheSubprovider({
                     vmErrorsOnRPCResponse: shouldThrowErrorsOnGanacheRPCResponse,
+                    db_path: config.ganacheDatabasePath,
                     gasLimit: constants.GAS_LIMIT,
                     logger,
                     verbose: env.parseBoolean(EnvVars.VerboseGanache),

packages/migrations/.gitignore

@@ -0,0 +1,2 @@
+*.zip
+0x_ganache_snapshot

packages/migrations/Dockerfile

@@ -0,0 +1,15 @@
+FROM mhart/alpine-node:10
+
+WORKDIR /usr/src/app
+
+RUN npm install -g ganache-cli@6.1.6
+
+ENV MNEMONIC "concert load couple harbor equip island argue ramp clarify fence smart topic"
+ENV NETWORK_ID 50
+ENV VERSION "latest"
+ENV SNAPSHOT_HOST "http://ganache-snapshots.0x.org.s3-website.us-east-2.amazonaws.com"
+ENV SNAPSHOT_NAME "0x_ganache_snapshot"
+EXPOSE 8545
+
+CMD [ "sh", "-c", "wget $SNAPSHOT_HOST/$SNAPSHOT_NAME-$VERSION.zip -O snapshot.zip && unzip snapshot.zip && ganache-cli --gasLimit 10000000 --db $SNAPSHOT_NAME --noVMErrorsOnRPCResponse -p 8545 --networkId \"$NETWORK_ID\" -m \"$MNEMONIC\" -h 0.0.0.0"]
+

packages/migrations/README.md

@@ -57,3 +57,44 @@ In order to migrate the V2 0x smart contracts to TestRPC/Ganache running at `htt
 ```bash
 yarn migrate:v2
 ```
+
+### Publish
+
+#### 0x Ganache Snapshot
+
+The 0x Ganache snapshot can be generated and published in this package. In order to build the snapshot for this version of migrations run:
+
+```bash
+yarn build:snapshot
+```
+
+This will run the migrations in Ganache and output a zip file to be uploaded to the s3 bucket. For example, after running this command you will have created `0x_ganache_snapshot-2.2.2.zip`. To publish the zip file to the s3 bucket run:
+
+```bash
+yarn publish:snapshot
+```
+
+This snapshot will now be publicly available at http://ganache-snapshots.0x.org.s3.amazonaws.com/0x_ganache_snapshot-latest.zip and also versioned with the package.json version.
+
+#### 0x Ganache Docker Image
+
+We also publish a simple docker image which downloads the latest snapshot, extracts and runs Ganache. This is not required to be built when migrations change as it always downloads and runs the latest zip file. If you have made changes to the Dockerfile then a publish of the image is required. To do this run:
+
+```bash
+yarn build:snapshot:docker
+yarn publish:snapshot:docker
+```
+
+The result is a published docker image to the 0xorg docker registry. To start the docker image run:
+
+```bash
+docker run -p 8545:8545 -ti 0xorg/ganache-cli:latest
+```
+
+This will pull the latest zip in the s3 bucket, extract and start Ganache with the snapshot.
+
+In the event you need a specific version of the published Ganache snapshot run the following specifying the VERSION environment variable:
+
+```bash
+docker run -e VERSION=2.2.2 -p 8545:8545 -ti 0xorg/ganache-cli:latest
+```

packages/migrations/package.json

@@ -10,13 +10,22 @@
     "scripts": {
         "build": "tsc -b",
         "build:ci": "yarn build",
-        "clean": "shx rm -rf lib",
+        "clean": "shx rm -rf lib ${npm_package_config_snapshot_name} ${npm_package_config_snapshot_name}-*.zip",
         "lint": "tslint --format stylish --project .",
         "migrate:v2": "run-s build script:migrate:v2",
+        "migrate:v2:snapshot": "run-s build script:migrate:v2:snapshot",
         "script:migrate:v2": "node ./lib/migrate.js",
-        "docs:json": "typedoc --excludePrivate --excludeExternals --target ES5 --tsconfig typedoc-tsconfig.json --json $JSON_FILE_PATH $PROJECT_FILES"
+        "script:migrate:v2:snapshot": "node ./lib/migrate_snapshot.js",
+        "docs:json": "typedoc --excludePrivate --excludeExternals --target ES5 --tsconfig typedoc-tsconfig.json --json $JSON_FILE_PATH $PROJECT_FILES",
+        "build:snapshot": "rm -rf ${npm_package_config_snapshot_name} && yarn migrate:v2:snapshot && zip -r \"${npm_package_config_snapshot_name}-${npm_package_version}.zip\" ${npm_package_config_snapshot_name}",
+        "build:snapshot:docker": "docker build --tag ${npm_package_config_docker_snapshot_name}:${npm_package_version} --tag ${npm_package_config_docker_snapshot_name}:latest .",
+        "publish:snapshot": "aws s3 cp ${npm_package_config_snapshot_name}-${npm_package_version}.zip ${npm_package_config_s3_snapshot_bucket} && aws s3 cp ${npm_package_config_s3_snapshot_bucket}/${npm_package_config_snapshot_name}-${npm_package_version}.zip ${npm_package_config_s3_snapshot_bucket}/${npm_package_config_snapshot_name}-latest.zip",
+        "publish:snapshot:docker": "docker push ${npm_package_config_docker_snapshot_name}:latest"
     },
     "config": {
+        "s3_snapshot_bucket": "s3://ganache-snapshots.0x.org",
+        "docker_snapshot_name": "0xorg/ganache-cli",
+        "snapshot_name": "0x_ganache_snapshot",
         "postpublish": {
             "assets": []
         }

packages/migrations/src/migrate_snapshot.ts

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+import { devConstants, web3Factory } from '@0x/dev-utils';
+import { logUtils } from '@0x/utils';
+import { Provider } from 'ethereum-types';
+import * as fs from 'fs';
+import * as _ from 'lodash';
+import * as path from 'path';
+
+import { runMigrationsAsync } from './migration';
+
+(async () => {
+    let providerConfigs;
+    let provider: Provider;
+    let txDefaults;
+    const packageJsonPath = path.join(__dirname, '..', 'package.json');
+    const packageJsonString = fs.readFileSync(packageJsonPath, 'utf8');
+    const packageJson = JSON.parse(packageJsonString);
+    if (_.isUndefined(packageJson.config) || _.isUndefined(packageJson.config.snapshot_name)) {
+        throw new Error(`Did not find 'snapshot_name' key in package.json config`);
+    }
+
+    providerConfigs = { shouldUseInProcessGanache: true, ganacheDatabasePath: packageJson.config.snapshot_name };
+    provider = web3Factory.getRpcProvider(providerConfigs);
+    txDefaults = {
+        from: devConstants.TESTRPC_FIRST_ADDRESS,
+    };
+    await runMigrationsAsync(provider, txDefaults);
+    process.exit(0);
+})().catch(err => {
+    logUtils.log(err);
+    process.exit(1);
+});

python-packages/order_utils/setup.py

@@ -137,14 +137,8 @@ class GanacheCommand(distutils.command.build_py.build_py):
     def run(self):
         """Run ganache."""
         cmd_line = (
-            "docker run -d -p 8545:8545 0xorg/ganache-cli --gasLimit"
+            "docker run -d -p 8545:8545 0xorg/ganache-cli:2.2.2"
-            + " 10000000 --db /snapshot --noVMErrorsOnRPCResponse -p 8545"
-            + " --networkId 50 -m"
         ).split()
-        cmd_line.append(
-            "concert load couple harbor equip island argue ramp clarify fence"
-            + " smart topic"
-        )
         subprocess.call(cmd_line)  # nosec