local_message_signer middleware for web3.py (#1713)

python-packages/cmd_pkgs_in_dep_order.py

@@ -15,6 +15,7 @@ PACKAGE_DEPENDENCY_LIST = [
     "json_schemas",
     "sra_client",
     "order_utils",
+    "middlewares",
     "contract_demo"
 ]
 

python-packages/middlewares/.pylintrc

@@ -0,0 +1,3 @@
+[MESSAGES CONTROL]
+disable=C0330,line-too-long,fixme,too-few-public-methods,too-many-ancestors
+# C0330 is "bad hanging indent". we use indents per `black`.

python-packages/middlewares/CHANGELOG.json

@@ -0,0 +1,11 @@
+[
+    {
+        "version": "1.0.0",
+        "changes": [
+            {
+                "note": "Initial publish",
+                "pr": 1713
+            }
+        ]
+    }
+]

python-packages/middlewares/README.md

@@ -0,0 +1,46 @@
+## 0x-middlewares
+
+Web3 middlewares for 0x applications.
+
+Read the [documentation](http://0x-middlewares-py.s3-website-us-east-1.amazonaws.com/)
+
+## Installing
+
+```bash
+pip install 0x-middlewares
+```
+
+## Contributing
+
+We welcome improvements and fixes from the wider community! To report bugs within this package, please create an issue in this repository.
+
+Please read our [contribution guidelines](../../CONTRIBUTING.md) before getting started.
+
+### Install Code and Dependencies
+
+Ensure that you have installed Python >=3.6 and Docker. Then:
+
+```bash
+pip install -e .[dev]
+```
+
+### Test
+
+Tests depend on running a local ethereum JSON-RPC server. For convenience, a docker container is provided that has ganache-cli.
+A shortcut is provided to run that docker container: `./setup.py ganache`. With that running, the tests can be run with `./setup.py test`.
+
+### Clean
+
+`./setup.py clean --all`
+
+### Lint
+
+`./setup.py lint`
+
+### Build Documentation
+
+`./setup.py build_sphinx`
+
+### More
+
+See `./setup.py --help-commands` for more info.

python-packages/middlewares/setup.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+
+"""setuptools module for middlewares package."""
+
+import subprocess  # nosec
+from shutil import rmtree
+from os import environ, path
+from sys import argv
+
+from distutils.command.clean import clean
+import distutils.command.build_py
+from setuptools import find_packages, setup
+from setuptools.command.test import test as TestCommand
+
+
+class TestCommandExtension(TestCommand):
+    """Run pytest tests."""
+
+    def run_tests(self):
+        """Invoke pytest."""
+        import pytest
+
+        exit(pytest.main(["--doctest-modules"]))
+
+
+class LintCommand(distutils.command.build_py.build_py):
+    """Custom setuptools command class for running linters."""
+
+    description = "Run linters"
+
+    def run(self):
+        """Run linter shell commands."""
+        lint_commands = [
+            # formatter:
+            "black --line-length 79 --check --diff src test setup.py".split(),
+            # style guide checker (formerly pep8):
+            "pycodestyle src test setup.py".split(),
+            # docstring style checker:
+            "pydocstyle src test setup.py".split(),
+            # static type checker:
+            "mypy src test setup.py".split(),
+            # security issue checker:
+            "bandit -r src ./setup.py".split(),
+            # general linter:
+            "pylint src test setup.py".split(),
+            # pylint takes relatively long to run, so it runs last, to enable
+            # fast failures.
+        ]
+
+        # tell mypy where to find interface stubs for 3rd party libs
+        environ["MYPYPATH"] = path.join(
+            path.dirname(path.realpath(argv[0])), "stubs"
+        )
+
+        for lint_command in lint_commands:
+            print(
+                "Running lint command `", " ".join(lint_command).strip(), "`"
+            )
+            subprocess.check_call(lint_command)  # nosec
+
+
+class CleanCommandExtension(clean):
+    """Custom command to do custom cleanup."""
+
+    def run(self):
+        """Run the regular clean, followed by our custom commands."""
+        super().run()
+        rmtree("dist", ignore_errors=True)
+        rmtree(".mypy_cache", ignore_errors=True)
+        rmtree(".tox", ignore_errors=True)
+        rmtree(".pytest_cache", ignore_errors=True)
+        rmtree("src/0x_middlewares.egg-info", ignore_errors=True)
+
+
+class TestPublishCommand(distutils.command.build_py.build_py):
+    """Custom command to publish to test.pypi.org."""
+
+    description = (
+        "Publish dist/* to test.pypi.org." "Run sdist & bdist_wheel first."
+    )
+
+    def run(self):
+        """Run twine to upload to test.pypi.org."""
+        subprocess.check_call(  # nosec
+            (
+                "twine upload --repository-url https://test.pypi.org/legacy/"
+                + " --verbose dist/*"
+            ).split()
+        )
+
+
+class PublishCommand(distutils.command.build_py.build_py):
+    """Custom command to publish to pypi.org."""
+
+    description = "Publish dist/* to pypi.org. Run sdist & bdist_wheel first."
+
+    def run(self):
+        """Run twine to upload to pypi.org."""
+        subprocess.check_call("twine upload dist/*".split())  # nosec
+
+
+class PublishDocsCommand(distutils.command.build_py.build_py):
+    """Custom command to publish docs to S3."""
+
+    description = (
+        "Publish docs to "
+        + "http://0x-middlewares-py.s3-website-us-east-1.amazonaws.com/"
+    )
+
+    def run(self):
+        """Run npm package `discharge` to build & upload docs."""
+        subprocess.check_call("discharge deploy".split())  # nosec
+
+
+class GanacheCommand(distutils.command.build_py.build_py):
+    """Custom command to publish to pypi.org."""
+
+    description = "Run ganache daemon to support tests."
+
+    def run(self):
+        """Run ganache."""
+        cmd_line = (
+            "docker run -d -p 8545:8545 0xorg/ganache-cli:2.2.2"
+        ).split()
+        subprocess.call(cmd_line)  # nosec
+
+
+with open("README.md", "r") as file_handle:
+    README_MD = file_handle.read()
+
+
+setup(
+    name="0x-middlewares",
+    version="1.0.1",
+    description="Web3 middlewares for 0x applications",
+    long_description=README_MD,
+    long_description_content_type="text/markdown",
+    url="https://github.com/0xproject/0x-monorepo/python-packages/middlewares",
+    author="Michael Huang",
+    author_email="michaelhly@users.noreply.github.com",
+    cmdclass={
+        "clean": CleanCommandExtension,
+        "lint": LintCommand,
+        "test": TestCommandExtension,
+        "test_publish": TestPublishCommand,
+        "publish": PublishCommand,
+        "publish_docs": PublishDocsCommand,
+        "ganache": GanacheCommand,
+    },
+    install_requires=[
+        "eth-account",
+        "eth-keys",
+        "hexbytes",
+        "hypothesis>=3.31.2",  # HACK! this is web3's dependency!
+        # above works around https://github.com/ethereum/web3.py/issues/1179
+        "mypy_extensions",
+    ],
+    extras_require={
+        "dev": [
+            "0x-contract-addresses",
+            "0x-order-utils",
+            "0x-web3",
+            "bandit",
+            "black",
+            "coverage",
+            "coveralls",
+            "eth_utils",
+            "mypy",
+            "mypy_extensions",
+            "pycodestyle",
+            "pydocstyle",
+            "pylint",
+            "pytest",
+            "sphinx",
+            "sphinx-autodoc-typehints",
+            "tox",
+            "twine",
+        ]
+    },
+    python_requires=">=3.6, <4",
+    package_data={"zero_ex.middlewares": ["py.typed"]},
+    package_dir={"": "src"},
+    license="Apache 2.0",
+    keywords=(
+        "ethereum cryptocurrency 0x decentralized blockchain dex exchange"
+    ),
+    namespace_packages=["zero_ex"],
+    packages=find_packages("src"),
+    classifiers=[
+        "Development Status :: 2 - Pre-Alpha",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Financial and Insurance Industry",
+        "License :: OSI Approved :: Apache Software License",
+        "Natural Language :: English",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Topic :: Internet :: WWW/HTTP",
+        "Topic :: Office/Business :: Financial",
+        "Topic :: Other/Nonlisted Topic",
+        "Topic :: Security :: Cryptography",
+        "Topic :: Software Development :: Libraries",
+        "Topic :: Utilities",
+    ],
+    zip_safe=False,  # required per mypy
+    command_options={
+        "build_sphinx": {
+            "source_dir": ("setup.py", "src"),
+            "build_dir": ("setup.py", "build/docs"),
+        }
+    },
+)

python-packages/middlewares/src/conf.py

@@ -0,0 +1,55 @@
+"""Configuration file for the Sphinx documentation builder."""
+
+# Reference: http://www.sphinx-doc.org/en/master/config
+
+from typing import List
+import pkg_resources
+
+
+# pylint: disable=invalid-name
+# because these variables are not named in upper case, as globals should be.
+
+project = "0x-middlewares"
+# pylint: disable=redefined-builtin
+copyright = "2019, ZeroEx, Intl."
+author = "Michael Hwang"
+version = pkg_resources.get_distribution("0x-middlewares").version
+release = ""  # The full version, including alpha/beta/rc tags
+
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.doctest",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.coverage",
+    "sphinx.ext.viewcode",
+    "sphinx_autodoc_typehints",
+]
+
+templates_path = ["doc_templates"]
+
+source_suffix = ".rst"
+# eg: source_suffix = [".rst", ".md"]
+
+master_doc = "index"  # The master toctree document.
+
+language = None
+
+exclude_patterns: List[str] = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+html_theme = "alabaster"
+
+html_static_path = ["doc_static"]
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "middlewarespydoc"
+
+# -- Extension configuration:
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {"https://docs.python.org/": None}

python-packages/middlewares/src/index.rst

@@ -0,0 +1,25 @@
+.. source for the sphinx-generated build/docs/web/index.html
+
+Python zero_ex.middlewares
+==========================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+.. automodule:: zero_ex.middlewares
+   :members:
+
+zero_ex.middlewares.local_message_signer
+----------------------------------------
+
+.. automodule:: zero_ex.middlewares.local_message_signer
+   :members:
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

python-packages/middlewares/src/zero_ex/__init__.py

@@ -0,0 +1,2 @@
+"""0x Python API."""
+__import__("pkg_resources").declare_namespace(__name__)

python-packages/middlewares/src/zero_ex/middlewares/__init__.py

@@ -0,0 +1 @@
+"""Web3 middlewares for 0x applications."""

python-packages/middlewares/src/zero_ex/middlewares/local_message_signer.py

@@ -0,0 +1,109 @@
+"""Middleware that captures all 'eth_sign' requests to the JSON-RPC-Server.
+
+An adaptation of the signing middleware from `web3.py
+<https://github.com/ethereum/web3.py/blob/master/web3/middleware/signing.py>`_.
+This middleware intercepts all 'eth_sign' requests to
+an ethereum JSON RPC-Server and signs messages with a local private key.
+"""
+
+from functools import singledispatch
+from typing import Dict, List, Set, Tuple, Union
+from eth_account import Account, messages
+from eth_account.local import LocalAccount
+from eth_keys.datatypes import PrivateKey
+from hexbytes import HexBytes
+
+
+@singledispatch
+def _to_account(private_key_or_account):
+    """Get a `LocalAccount` instance from a private_key or a `LocalAccount`.
+
+    Note that this function is overloaded based on the type on input. This
+    implementation is the base case where none of the supported types are
+    matched and we throw an exception.
+    """
+    raise TypeError(
+        "key must be one of the types:"
+        "eth_keys.datatype.PrivateKey, "
+        "eth_account.local.LocalAccount, "
+        "or raw private key as a hex string or byte string. "
+        "Was of type {0}".format(type(private_key_or_account))
+    )
+
+
+def _private_key_to_account(private_key):
+    """Get the account associated with the private key."""
+    if isinstance(private_key, PrivateKey):
+        private_key = private_key.to_hex()
+    else:
+        private_key = HexBytes(private_key).hex()
+    return Account().privateKeyToAccount(private_key)
+
+
+_to_account.register(LocalAccount, lambda x: x)
+_to_account.register(PrivateKey, _private_key_to_account)
+_to_account.register(str, _private_key_to_account)
+_to_account.register(bytes, _private_key_to_account)
+
+
+def construct_local_message_signer(
+    private_key_or_account: Union[
+        Union[LocalAccount, PrivateKey, str],
+        List[Union[LocalAccount, PrivateKey, str]],
+        Tuple[Union[LocalAccount, PrivateKey, str]],
+        Set[Union[LocalAccount, PrivateKey, str]],
+    ]
+):
+    """Construct a local messager signer middleware.
+
+    :param private_key_or_account: a single private key or a tuple,
+        list, or set of private keys. Keys can be any of the following
+        formats:
+
+            - An `eth_account.LocalAccount` object
+            - An `eth_keys.PrivateKey` object
+            - A raw private key as a hex `string` or `bytes`
+    :returns: callable local_message_signer_middleware
+
+    :Example:
+
+    >>> from web3 import Web3
+    >>> from zero_ex.middlewares.local_message_signer import (
+    ...     construct_local_message_signer)
+    >>> WEB3_RPC_URL="https://mainnet.infura.io/v3/INFURA_API_KEY"
+    >>> PRIVATE_KEY=(
+    ...     "f2f48ee19680706196e2e339e5da3491186e0c4c5030670656b0e0164837257d")
+    >>> web3_instance = Web3.HTTPProvider(WEB3_RPC_URL)
+    >>> web3_instance.middlewares.add(
+    ...     construct_local_message_signer(PRIVATE_KEY))
+
+    """
+    if not isinstance(private_key_or_account, (list, tuple, set)):
+        private_key_or_account = [private_key_or_account]
+    accounts = [_to_account(pkoa) for pkoa in private_key_or_account]
+    address_to_accounts: Dict[str, LocalAccount] = {
+        account.address: account for account in accounts
+    }
+
+    def local_message_signer_middleware(
+        make_request, web3
+    ):  # pylint: disable=unused-argument
+        def middleware(method, params):
+            if method != "eth_sign":
+                return make_request(method, params)
+            account_address, message = params[:2]
+            account = address_to_accounts[account_address]
+            # We will assume any string which looks like a hex is expected
+            # to be converted to hex. Non-hexable strings are forcibly
+            # converted by encoding them to utf-8
+            try:
+                message = HexBytes(message)
+            except Exception:  # pylint: disable=broad-except
+                message = HexBytes(message.encode("utf-8"))
+            msg_hash_hexbytes = messages.defunct_hash_message(message)
+            ec_signature = account.signHash(msg_hash_hexbytes)
+            return {"result": ec_signature.signature}
+
+        return middleware
+
+    return local_message_signer_middleware

python-packages/middlewares/stubs/distutils/command/clean.pyi

@@ -0,0 +1,7 @@
+from distutils.core import Command
+
+class clean(Command):
+    def initialize_options(self: clean) -> None: ...
+    def finalize_options(self: clean) -> None: ...
+    def run(self: clean) -> None: ...
+    ...

python-packages/middlewares/stubs/eth_account/__init__.pyi

@@ -0,0 +1 @@
+class Account: ...

python-packages/middlewares/stubs/eth_account/local.pyi

@@ -0,0 +1 @@
+class LocalAccount: ...

python-packages/middlewares/stubs/eth_keys/datatypes.pyi

@@ -0,0 +1 @@
+class PrivateKey: ...

python-packages/middlewares/stubs/eth_utils/__init__.pyi

@@ -0,0 +1,3 @@
+from typing import Union
+
+def to_checksum_address(value: Union[str, bytes]) -> str: ...

python-packages/middlewares/stubs/hexbytes/HexBytes.pyi

@@ -0,0 +1 @@
+class HexBytes: ...

python-packages/middlewares/stubs/pytest/raises.pyi

@@ -0,0 +1 @@
+def raises(exception: Exception) -> ExceptionInfo: ...

python-packages/middlewares/stubs/setuptools/__init__.pyi

@@ -0,0 +1,8 @@
+from distutils.dist import Distribution
+from typing import Any, List
+
+def setup(**attrs: Any) -> Distribution: ...
+
+class Command: ...
+
+def find_packages(where: str) -> List[str]: ...

python-packages/middlewares/stubs/setuptools/command/test.pyi

@@ -0,0 +1,3 @@
+from setuptools import Command
+
+class test(Command): ...

python-packages/middlewares/stubs/web3/__init__.pyi

@@ -0,0 +1,8 @@
+from typing import Dict, Optional, Union
+
+from web3.utils import datatypes
+from web3.providers.base import BaseProvider
+
+class Web3:
+    class HTTPProvider(BaseProvider): ...
+    def __init__(self, provider: BaseProvider) -> None: ...

python-packages/middlewares/stubs/web3/providers/base.pyi

@@ -0,0 +1 @@
+class BaseProvider: ...

python-packages/middlewares/test/__init__.py

@@ -0,0 +1 @@
+"""Tests of zero_x.middlewares."""

python-packages/middlewares/test/test_local_message_signer.py

@@ -0,0 +1,40 @@
+"""Tests of 0x.middlewares.local_message_signer."""
+
+from eth_utils import to_checksum_address
+from web3 import Web3
+from zero_ex.contract_addresses import NETWORK_TO_ADDRESSES, NetworkId
+from zero_ex.middlewares.local_message_signer import (
+    construct_local_message_signer,
+)
+from zero_ex.order_utils import (
+    generate_order_hash_hex,
+    is_valid_signature,
+    make_empty_order,
+    sign_hash,
+)
+
+
+def test_local_message_signer__sign_order():
+    """Test signing order with the local_message_signer middleware"""
+    expected_signature = (
+        "0x1cd17d75b891accf16030c572a64cf9e7955de63bcafa5b084439cec630ade2d7"
+        "c00f47a2f4d5b6a4508267bf4b8527100bd97cf1af9984c0a58e42d25b13f4f0a03"
+    )
+    address = "0x5409ED021D9299bf6814279A6A1411A7e866A631"
+    exchange = NETWORK_TO_ADDRESSES[NetworkId.GANACHE].exchange
+    private_key = (
+        "f2f48ee19680706196e2e339e5da3491186e0c4c5030670656b0e0164837257d"
+    )
+    web3_rpc_url = "http://127.0.0.1:8545"
+    web3_instance = Web3.HTTPProvider(web3_rpc_url)
+    web3_instance.middlewares.add(construct_local_message_signer(private_key))
+    order = make_empty_order()
+    order_hash = generate_order_hash_hex(order, exchange)
+    signature = sign_hash(
+        web3_instance, to_checksum_address(address), order_hash
+    )
+    assert signature == expected_signature
+    is_valid = is_valid_signature(
+        web3_instance, order_hash, signature, address
+    )[0]
+    assert is_valid is True

python-packages/middlewares/tox.ini

@@ -0,0 +1,25 @@
+# tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+envlist = py37
+
+[testenv]
+commands =
+    pip install -e .[dev]
+    python setup.py test
+
+[testenv:run_tests_against_test_deployment]
+commands =
+    # install dependencies from real PyPI
+    pip install eth-account eth-keys hexbytes hypothesis>=3.31.2 mypy_extensions
+    # install package-under-test from test PyPI
+    pip install --index-url https://test.pypi.org/legacy/ 0x-middlewares 
+    pytest test
+
+[testenv:run_tests_against_deployment]
+commands =
+    pip install 0x-middlewares
+    pytest test