feat[tool]: support storage layouts via json and .vyz inputs (#4370)

This commit adds support for overriding the storage layout using
`solc_json` and archive inputs, and consequently adding the storage
layout if it was provided to these formats as output. This makes it
possible for verifiers to verify code compiled with a storage layout
override with no changes on their end.

A design decision was made to have the storage layout override affect
the integrity hash. This is so you can tell that a contract was
compiled with storage layout override (even if it does not affect the
bytecode).

---------

Co-authored-by: Charles Cooper <[email protected]>

docs/compiling-a-contract.rst

@@ -106,7 +106,7 @@ Online Compilers
 Try VyperLang!
 -----------------
 
-`Try VyperLang! <https://try.vyperlang.org>`_ is a JupterHub instance hosted by the Vyper team as a sandbox for developing and testing contracts in Vyper. It requires github for login, and supports deployment via the browser.
+`Try VyperLang! <https://try.vyperlang.org>`_ is a JupyterHub instance hosted by the Vyper team as a sandbox for developing and testing contracts in Vyper. It requires github for login, and supports deployment via the browser.
 
 Remix IDE
 ---------
@@ -203,7 +203,7 @@ The following is a list of supported EVM versions, and changes in the compiler i
 Integrity Hash
 ==============
 
-To help tooling detect whether two builds are the same, Vyper provides the ``-f integrity`` output, which outputs the integrity hash of a contract. The integrity hash is recursively defined as the sha256 of the source code with the integrity hashes of its dependencies (imports).
+To help tooling detect whether two builds are the same, Vyper provides the ``-f integrity`` output, which outputs the integrity hash of a contract. The integrity hash is recursively defined as the sha256 of the source code with the integrity hashes of its dependencies (imports) and storage layout overrides (if provided).
 
 .. _vyper-archives:
 
@@ -219,15 +219,17 @@ A Vyper archive is a compileable bundle of input sources and settings. Technical
     ├── compilation_targets
     ├── compiler_version
     ├── integrity
+    ├── settings.json
     ├── searchpaths
-    └── settings.json
+    └── storage_layout.json [OPTIONAL]
 
 * ``cli_settings.txt`` is a text representation of the settings that were used on the compilation run that generated this archive.
 * ``compilation_targets`` is a newline separated list of compilation targets. Currently only one compilation is supported
 * ``compiler_version`` is a text representation of the compiler version used to generate this archive
 * ``integrity`` is the :ref:`integrity hash <integrity-hash>` of the input contract
 * ``searchpaths`` is a newline-separated list of the search paths used on this compilation run
 * ``settings.json`` is a json representation of the settings used on this compilation run. It is 1:1 with ``cli_settings.txt``, but both are provided as they are convenient for different workflows (typically, manually vs automated).
+* ``storage_layout.json`` is a json representation of the storage layout overrides to be used on this compilation run. It is optional.
 
 A Vyper archive file can be produced by requesting the ``-f archive`` output format. The compiler can also produce the archive in base64 encoded form using the ``--base64`` flag. The Vyper compiler can accept both ``.vyz`` and base64-encoded Vyper archives directly as input.
 
@@ -281,6 +283,14 @@ The following example describes the expected input format of ``vyper-json``. (Co
             }
         },
         // Optional
+        // Storage layout overrides for the contracts that are compiled
+        "storage_layout_overrides": {
+            "contracts/foo.vy": {
+                "a": {"type": "uint256", "slot": 1, "n_slots": 1},
+                "b": {"type": "uint256", "slot": 0, "n_slots": 1},
+            }
+        },
+        // Optional
         "settings": {
             "evmVersion": "cancun",  // EVM version to compile for. Can be london, paris, shanghai or cancun (default).
             // optional, optimization mode
@@ -364,6 +374,13 @@ The following example describes the output format of ``vyper-json``. Comments ar
             "formattedMessage": "line 5:11 Unsupported type conversion: int128 to bool"
             }
         ],
+        // Optional: not present if there are no storage layout overrides
+        "storage_layout_overrides": {
+            "contracts/foo.vy": {
+                "a": {"type": "uint256", "slot": 1, "n_slots": 1},
+                "b": {"type": "uint256", "slot": 0, "n_slots": 1},
+            }
+        },
         // This contains the file-level outputs. Can be limited/filtered by the outputSelection settings.
         "sources": {
             "source_file.vy": {

tests/unit/cli/storage_layout/test_storage_layout_overrides.py

@@ -2,6 +2,7 @@
 
 import pytest
 
+from vyper.cli.vyper_json import compile_json
 from vyper.compiler import compile_code
 from vyper.evm.opcodes import version_check
 from vyper.exceptions import StorageLayoutException
@@ -13,7 +14,7 @@ def test_storage_layout_overrides():
 b: uint256"""
 
     storage_layout_overrides = {
-        "a": {"type": "uint256", "slot": 1, "n_slots": 1},
+        "a": {"type": "uint256", "slot": 5, "n_slots": 1},
         "b": {"type": "uint256", "slot": 0, "n_slots": 1},
     }
 
@@ -26,6 +27,31 @@ def test_storage_layout_overrides():
     assert out["layout"] == expected_output
 
 
+def test_storage_layout_overrides_json():
+    code = """
+a: uint256
+b: uint256"""
+
+    storage_layout_overrides = {
+        "a": {"type": "uint256", "slot": 5, "n_slots": 1},
+        "b": {"type": "uint256", "slot": 0, "n_slots": 1},
+    }
+
+    input_json = {
+        "language": "Vyper",
+        "sources": {"contracts/foo.vy": {"content": code}},
+        "storage_layout_overrides": {"contracts/foo.vy": storage_layout_overrides},
+        "settings": {"outputSelection": {"*": ["*"]}},
+    }
+
+    out = compile_code(
+        code, output_formats=["layout"], storage_layout_override=storage_layout_overrides
+    )
+    assert (
+        compile_json(input_json)["contracts"]["contracts/foo.vy"]["foo"]["layout"] == out["layout"]
+    )
+
+
 def test_storage_layout_for_more_complex():
     code = """
 foo: HashMap[address, uint256]

tests/unit/cli/vyper_compile/test_compile_files.py

@@ -1,18 +1,27 @@
 import contextlib
+import json
 import sys
+import warnings
 import zipfile
 from pathlib import Path
 
 import pytest
 
+from vyper.cli.compile_archive import compiler_data_from_zip
 from vyper.cli.vyper_compile import compile_files
-from vyper.cli.vyper_json import compile_json
+from vyper.cli.vyper_json import compile_from_input_dict, compile_json
 from vyper.compiler import INTERFACE_OUTPUT_FORMATS, OUTPUT_FORMATS
 from vyper.compiler.input_bundle import FilesystemInputBundle
 from vyper.compiler.output_bundle import OutputBundle
 from vyper.compiler.phases import CompilerData
 from vyper.utils import sha256sum
 
+TAMPERED_INTEGRITY_SUM = sha256sum("tampered integrity sum")
+
+INTEGRITY_WARNING = f"Mismatched integrity sum! Expected {TAMPERED_INTEGRITY_SUM}"
+INTEGRITY_WARNING += " but got {integrity}."  # noqa: FS003
+INTEGRITY_WARNING += " (This likely indicates a corrupted archive)"
+
 
 def test_combined_json_keys(chdir_tmp_path, make_file):
     make_file("bar.vy", "")
@@ -298,6 +307,9 @@ def foo() -> uint256:
 import lib
 import jsonabi
 
+a: uint256
+b: uint256
+
 @external
 def foo() -> uint256:
     return lib.foo()
@@ -306,28 +318,52 @@ def foo() -> uint256:
 def bar(x: uint256) -> uint256:
     return extcall jsonabi(msg.sender).test_json(x)
     """
+    storage_layout_overrides = {
+        "a": {"type": "uint256", "n_slots": 1, "slot": 5},
+        "b": {"type": "uint256", "n_slots": 1, "slot": 0},
+    }
+    storage_layout_source = json.dumps(storage_layout_overrides)
+
     tmpdir = tmp_path_factory.mktemp("fake-package")
     with open(tmpdir / "lib.vy", "w") as f:
         f.write(library_source)
     with open(tmpdir / "jsonabi.json", "w") as f:
         f.write(json_source)
+    with open(tmpdir / "layout.json", "w") as f:
+        f.write(storage_layout_source)
 
     contract_file = make_file("contract.vy", contract_source)
 
-    return (tmpdir, tmpdir / "lib.vy", tmpdir / "jsonabi.json", contract_file)
+    contract_hash = sha256sum(contract_source)
+    library_hash = sha256sum(library_source)
+    jsonabi_hash = sha256sum(json_source)
+    resolved_imports_hash = sha256sum(contract_hash + sha256sum(library_hash) + jsonabi_hash)
+    storage_layout_hash = sha256sum(storage_layout_source)
+    expected_integrity = sha256sum(storage_layout_hash + resolved_imports_hash)
+
+    return (
+        tmpdir,
+        tmpdir / "lib.vy",
+        tmpdir / "jsonabi.json",
+        tmpdir / "layout.json",
+        contract_file,
+        expected_integrity,
+    )
 
 
 def test_import_sys_path(input_files):
-    tmpdir, _, _, contract_file = input_files
+    tmpdir, _, _, _, contract_file, _ = input_files
     with mock_sys_path(tmpdir):
         assert compile_files([contract_file], ["combined_json"]) is not None
 
 
 def test_archive_output(input_files):
-    tmpdir, _, _, contract_file = input_files
+    tmpdir, library_file, jsonabi_file, storage_layout_path, contract_file, integrity = input_files
     search_paths = [".", tmpdir]
 
-    s = compile_files([contract_file], ["archive"], paths=search_paths)
+    s = compile_files(
+        [contract_file], ["archive"], paths=search_paths, storage_layout_paths=[storage_layout_path]
+    )
     archive_bytes = s[contract_file]["archive"]
 
     archive_path = Path("foo.zip")
@@ -337,13 +373,28 @@ def test_archive_output(input_files):
     assert zipfile.is_zipfile(archive_path)
 
     # compare compiling the two input bundles
-    out = compile_files([contract_file], ["integrity", "bytecode"], paths=search_paths)
-    out2 = compile_files([archive_path], ["integrity", "bytecode"])
+    out = compile_files(
+        [contract_file],
+        ["integrity", "bytecode", "layout"],
+        paths=search_paths,
+        storage_layout_paths=[storage_layout_path],
+    )
+    out2 = compile_files([archive_path], ["integrity", "bytecode", "layout"])
     assert out[contract_file] == out2[archive_path]
 
+    # tamper with the integrity sum
+    archive_compiler_data = compiler_data_from_zip(archive_path, None, False)
+    archive_compiler_data.expected_integrity_sum = TAMPERED_INTEGRITY_SUM
+
+    with warnings.catch_warnings(record=True) as w:
+        assert archive_compiler_data.integrity_sum is not None
+
+    assert len(w) == 1, [s.message for s in w]
+    assert str(w[0].message).startswith(INTEGRITY_WARNING.format(integrity=integrity))
+
 
 def test_archive_b64_output(input_files):
-    tmpdir, _, _, contract_file = input_files
+    tmpdir, _, _, _, contract_file, _ = input_files
     search_paths = [".", tmpdir]
 
     out = compile_files(
@@ -362,7 +413,7 @@ def test_archive_b64_output(input_files):
 
 
 def test_archive_compile_options(input_files):
-    tmpdir, _, _, contract_file = input_files
+    tmpdir, _, _, _, contract_file, _ = input_files
     search_paths = [".", tmpdir]
 
     options = ["abi_python", "json", "ast", "annotated_ast", "ir_json"]
@@ -417,7 +468,7 @@ def test_archive_compile_options(input_files):
 
 
 def test_compile_vyz_with_options(input_files):
-    tmpdir, _, _, contract_file = input_files
+    tmpdir, _, _, _, contract_file, _ = input_files
     search_paths = [".", tmpdir]
 
     for option in format_options:
@@ -446,7 +497,7 @@ def test_compile_vyz_with_options(input_files):
 
 
 def test_archive_compile_simultaneous_options(input_files):
-    tmpdir, _, _, contract_file = input_files
+    tmpdir, _, _, _, contract_file, _ = input_files
     search_paths = [".", tmpdir]
 
     for option in format_options:
@@ -461,40 +512,55 @@ def test_archive_compile_simultaneous_options(input_files):
 
 
 def test_solc_json_output(input_files):
-    tmpdir, _, _, contract_file = input_files
+    tmpdir, _, _, storage_layout_path, contract_file, integrity = input_files
     search_paths = [".", tmpdir]
 
-    out = compile_files([contract_file], ["solc_json"], paths=search_paths)
-
+    out = compile_files(
+        [contract_file],
+        ["solc_json"],
+        paths=search_paths,
+        storage_layout_paths=[storage_layout_path],
+    )
     json_input = out[contract_file]["solc_json"]
 
     # check that round-tripping solc_json thru standard json produces
     # the same as compiling directly
     json_out = compile_json(json_input)["contracts"]["contract.vy"]
     json_out_bytecode = json_out["contract"]["evm"]["bytecode"]["object"]
+    json_out_layout = json_out["contract"]["layout"]["storage_layout"]
 
-    out2 = compile_files([contract_file], ["integrity", "bytecode"], paths=search_paths)
+    out2 = compile_files(
+        [contract_file],
+        ["integrity", "bytecode", "layout"],
+        paths=search_paths,
+        storage_layout_paths=[storage_layout_path],
+    )
 
     assert out2[contract_file]["bytecode"] == json_out_bytecode
+    assert out2[contract_file]["layout"]["storage_layout"] == json_out_layout
+
+    # tamper with the integrity sum
+    json_input["integrity"] = TAMPERED_INTEGRITY_SUM
+    _, warn_data = compile_from_input_dict(json_input)
+
+    w = warn_data[Path("contract.vy")]
+    assert len(w) == 1, [s.message for s in w]
+    assert str(w[0].message).startswith(INTEGRITY_WARNING.format(integrity=integrity))
 
 
 # maybe this belongs in tests/unit/compiler?
 def test_integrity_sum(input_files):
-    tmpdir, library_file, jsonabi_file, contract_file = input_files
+    tmpdir, library_file, jsonabi_file, storage_layout_path, contract_file, integrity = input_files
     search_paths = [".", tmpdir]
 
-    out = compile_files([contract_file], ["integrity"], paths=search_paths)
-
-    with library_file.open() as f, contract_file.open() as g, jsonabi_file.open() as h:
-        library_contents = f.read()
-        contract_contents = g.read()
-        jsonabi_contents = h.read()
+    out = compile_files(
+        [contract_file],
+        ["integrity"],
+        paths=search_paths,
+        storage_layout_paths=[storage_layout_path],
+    )
 
-    contract_hash = sha256sum(contract_contents)
-    library_hash = sha256sum(library_contents)
-    jsonabi_hash = sha256sum(jsonabi_contents)
-    expected = sha256sum(contract_hash + sha256sum(library_hash) + jsonabi_hash)
-    assert out[contract_file]["integrity"] == expected
+    assert out[contract_file]["integrity"] == integrity
 
 
 # does this belong in tests/unit/compiler?