diff --git a/.github/actions/build-fixtures/action.yaml b/.github/actions/build-fixtures/action.yaml index 785b985ef44..f3b1d003c68 100644 --- a/.github/actions/build-fixtures/action.yaml +++ b/.github/actions/build-fixtures/action.yaml @@ -4,6 +4,15 @@ inputs: release_name: description: "Name of the fixture release" required: true + from_fork: + description: "Fill from this fork (inclusive). Empty for unsplit builds." + default: "" + until_fork: + description: "Fill until this fork (inclusive). Empty for unsplit builds." + default: "" + split_label: + description: "Label for this fork-range split. Empty for unsplit builds." + default: "" runs: using: "composite" steps: @@ -25,28 +34,54 @@ runs: with: type: ${{ steps.properties.outputs.evm-type }} - name: Install pigz for parallel tarball compression + if: inputs.split_label == '' shell: bash run: sudo apt-get install -y pigz - name: Generate fixtures using fill shell: bash run: | + IS_SPLIT="${{ inputs.split_label }}" + + if [ -n "$IS_SPLIT" ]; then + OUTPUT_ARG="--output=fixtures_${{ inputs.release_name }}" + FORK_ARGS="--generate-all-formats --from=${{ inputs.from_fork }} --until=${{ inputs.until_fork }}" + else + OUTPUT_ARG="--output=fixtures_${{ inputs.release_name }}.tar.gz" + FORK_ARGS="" + fi + + # Allow exit code 5 (NO_TESTS_COLLECTED) for fork ranges with no tests. + EXIT_CODE=0 if [ "${{ steps.evm-builder.outputs.impl }}" = "eels" ]; then - uv run fill -n ${{ steps.evm-builder.outputs.x-dist }} ${{ steps.properties.outputs.fill-params }} --output=fixtures_${{ inputs.release_name }}.tar.gz --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG + uv run fill -n ${{ steps.evm-builder.outputs.x-dist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? else - uv run fill -n ${{ steps.evm-builder.outputs.x-dist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} --output=fixtures_${{ inputs.release_name }}.tar.gz --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG + uv run fill -n ${{ steps.evm-builder.outputs.x-dist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? + fi + if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 5 ]; then + exit "$EXIT_CODE" fi - name: Generate Benchmark Genesis Files - if: contains(inputs.release_name, 'benchmark') + if: inputs.split_label == '' && contains(inputs.release_name, 'benchmark') uses: ./.github/actions/build-benchmark-genesis with: fixtures_path: fixtures_${{ inputs.release_name }}.tar.gz - name: Upload Benchmark Genesis Artifact - if: contains(inputs.release_name, 'benchmark') + if: inputs.split_label == '' && contains(inputs.release_name, 'benchmark') uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: benchmark_genesis_${{ inputs.release_name }} path: benchmark_genesis.tar.gz - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - name: Upload fixture tarball (unsplit) + if: inputs.split_label == '' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: fixtures_${{ inputs.release_name }} path: fixtures_${{ inputs.release_name }}.tar.gz + - name: Upload fixture directory (split) + if: inputs.split_label != '' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: fixtures__${{ inputs.split_label }} + include-hidden-files: true + path: fixtures_${{ inputs.release_name }}/ + if-no-files-found: ignore diff --git a/.github/configs/evm.yaml b/.github/configs/evm.yaml index 0912e751d64..c295d43bb86 100644 --- a/.github/configs/evm.yaml +++ b/.github/configs/evm.yaml @@ -1,8 +1,4 @@ -stable: - impl: eels - repo: null - ref: null -develop: +eels: impl: eels repo: null ref: null diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index 22063a82cac..eeb63f272f7 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -1,15 +1,12 @@ # Unless filling for special features, all features should fill for previous forks (starting from Frontier) too -stable: - evm-type: stable - fill-params: --until=Prague --fill-static-tests --ignore=tests/static/state_tests/stQuadraticComplexityTest - -develop: - evm-type: develop - fill-params: --until=BPO4 --fill-static-tests --ignore=tests/static/state_tests/stQuadraticComplexityTest +mainnet: + evm-type: eels + fill-params: --until=BPO2 benchmark: evm-type: benchmark fill-params: --fork=Osaka --gas-benchmark-values 1,5,10,30,60,100,150 ./tests/benchmark/compute --maxprocesses=30 --dist=worksteal + feature_only: true benchmark_fast: evm-type: benchmark @@ -17,6 +14,6 @@ benchmark_fast: feature_only: true bal: - evm-type: develop - fill-params: --fork=Amsterdam --fill-static-tests + evm-type: eels + fill-params: --fork=Amsterdam feature_only: true diff --git a/.github/configs/fork-ranges.yaml b/.github/configs/fork-ranges.yaml new file mode 100644 index 00000000000..feb5ed48e38 --- /dev/null +++ b/.github/configs/fork-ranges.yaml @@ -0,0 +1,21 @@ +# Shared fork ranges for splitting multi-fork releases across parallel runners. +# Features using --until are automatically split using applicable ranges. +# Features using --fork (single fork) are never split. +- label: pre-cancun + from: Frontier + until: Shanghai +- label: cancun + from: Cancun + until: Cancun +- label: prague + from: Prague + until: Prague +- label: osaka + from: Osaka + until: Osaka +- label: bpo + from: BPO1 + until: BPO2 +- label: amsterdam + from: Amsterdam + until: Amsterdam diff --git a/.github/scripts/create_release_tarball.py b/.github/scripts/create_release_tarball.py new file mode 100644 index 00000000000..1f4e7d47a2f --- /dev/null +++ b/.github/scripts/create_release_tarball.py @@ -0,0 +1,84 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# dependencies = [] +# /// +""" +Create a release tarball from a merged fixture directory. + +Archive all ``.json`` and ``.ini`` files under a ``fixtures/`` prefix, +matching the structure produced by +``execution_testing.cli.pytest_commands.plugins.shared.fixture_output``. + +Use ``pigz`` for parallel compression when available, otherwise fall +back to Python's built-in gzip. +""" + +import shutil +import subprocess +import sys +import tarfile +import warnings +from pathlib import Path + + +def create_tarball_with_pigz(source_dir: Path, output_path: Path) -> None: + """Create tarball using Python tarfile + pigz for parallel compression.""" + temp_tar = output_path.with_suffix("") # strip .gz + + with tarfile.open(temp_tar, "w") as tar: + for file in sorted(source_dir.rglob("*")): + if file.is_file() and file.suffix in {".json", ".ini"}: + arcname = Path("fixtures") / file.relative_to(source_dir) + tar.add(file, arcname=str(arcname)) + + subprocess.run( + ["pigz", "-f", str(temp_tar)], + check=True, + capture_output=True, + ) + + +def create_tarball_standard(source_dir: Path, output_path: Path) -> None: + """Create tarball using Python's tarfile module (single-threaded).""" + with tarfile.open(output_path, "w:gz") as tar: + for file in sorted(source_dir.rglob("*")): + if file.is_file() and file.suffix in {".json", ".ini"}: + arcname = Path("fixtures") / file.relative_to(source_dir) + tar.add(file, arcname=str(arcname)) + + +def main() -> None: + """Entry point.""" + if len(sys.argv) != 3: + print( + "Usage: create_release_tarball.py ", + file=sys.stderr, + ) + sys.exit(1) + + source_dir = Path(sys.argv[1]) + output_path = Path(sys.argv[2]) + + if not source_dir.is_dir(): + print(f"Error: '{source_dir}' is not a directory.", file=sys.stderr) + sys.exit(1) + + if shutil.which("pigz"): + try: + create_tarball_with_pigz(source_dir, output_path) + except (subprocess.CalledProcessError, OSError) as e: + warnings.warn( + f"pigz failed ({type(e).__name__}: {e}), falling back to gzip", + stacklevel=2, + ) + create_tarball_standard(source_dir, output_path) + else: + create_tarball_standard(source_dir, output_path) + + print(f"Created {output_path}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/generate_build_matrix.py b/.github/scripts/generate_build_matrix.py new file mode 100644 index 00000000000..de8b3b5144a --- /dev/null +++ b/.github/scripts/generate_build_matrix.py @@ -0,0 +1,160 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "pyyaml", +# ] +# /// +""" +Generate the build matrix for release fixture workflows. + +Read `.github/configs/feature.yaml` and emit a flat JSON build matrix +suitable for ``strategy.matrix`` in GitHub Actions. + +Features whose ``fill-params`` contain ``--until`` are split across the +shared fork ranges defined in `.github/configs/fork-ranges.yaml`. +Features using ``--fork`` (single fork) produce a single unsplit entry. +""" + +import json +import re +import sys +from pathlib import Path + +import yaml + +FEATURE_CONFIG = Path(".github/configs/feature.yaml") +FORK_RANGES_CONFIG = Path(".github/configs/fork-ranges.yaml") + +# Canonical fork ordering used to filter fork ranges per feature. +FORK_ORDER = [ + "Frontier", + "Homestead", + "DAOFork", + "TangerineWhistle", + "SpuriousDragon", + "Byzantium", + "Constantinople", + "Istanbul", + "MuirGlacier", + "Berlin", + "London", + "ArrowGlacier", + "GrayGlacier", + "Paris", + "Shanghai", + "Cancun", + "Prague", + "Osaka", + "BPO1", + "BPO2", + "Amsterdam", +] + +FORK_INDEX = {name: i for i, name in enumerate(FORK_ORDER)} + + +def load_config(path: Path) -> dict: + """Load and return the feature configuration.""" + with open(path) as f: + return yaml.safe_load(f) + + +def parse_until_fork(fill_params: str) -> str | None: + """ + Extract the ``--until`` value from fill-params. + + Return ``None`` when ``--fork`` is used instead (single-fork + feature that should not be split). + """ + if re.search(r"--fork\b", fill_params): + return None + m = re.search(r"--until[=\s]+(\S+)", fill_params) + return m.group(1) if m else None + + +def applicable_ranges(fork_ranges: list[dict], until_fork: str) -> list[dict]: + """ + Return fork ranges whose ``from`` is at or before *until_fork*. + + Clamp the last applicable range's ``until`` to *until_fork* so we + never fill beyond the feature's declared boundary. + """ + limit = FORK_INDEX[until_fork] + result = [] + for r in fork_ranges: + if FORK_INDEX[r["from"]] <= limit: + entry = dict(r) + if FORK_INDEX[r["until"]] > limit: + entry["until"] = until_fork + result.append(entry) + return result + + +def build_matrix( + feature: dict, name: str, fork_ranges: list[dict] +) -> tuple[list[dict], str]: + """ + Build the matrix for a single feature. + + Return (build_entries, combine_labels). Split features produce + one entry per fork range and a space-separated label string for + the combine step. Unsplit features produce a single entry with + empty labels. + """ + until = parse_until_fork(feature["fill-params"]) + if until and fork_ranges: + ranges = applicable_ranges(fork_ranges, until) + if len(ranges) > 1: + build = [ + { + "feature": name, + "label": r["label"], + "from_fork": r["from"], + "until_fork": r["until"], + } + for r in ranges + ] + labels = " ".join(r["label"] for r in ranges) + return build, labels + + return [ + { + "feature": name, + "label": "", + "from_fork": "", + "until_fork": "", + } + ], "" + + +def main() -> None: + """Entry point.""" + if len(sys.argv) != 2: + print( + "Usage: generate_build_matrix.py ", + file=sys.stderr, + ) + sys.exit(1) + + config = load_config(FEATURE_CONFIG) + fork_ranges = load_config(FORK_RANGES_CONFIG) or [] + name = sys.argv[1] + + if name not in config or not isinstance(config[name], dict): + print( + f"Error: feature '{name}' not found in {FEATURE_CONFIG}.", + file=sys.stderr, + ) + sys.exit(1) + + build, labels = build_matrix(config[name], name, fork_ranges) + + print(f"build_matrix={json.dumps(build)}") + print(f"feature_name={name}") + print(f"combine_labels={labels}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/merge_index_files.py b/.github/scripts/merge_index_files.py new file mode 100644 index 00000000000..ea41b299a10 --- /dev/null +++ b/.github/scripts/merge_index_files.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +""" +Merge multiple .meta/index.json files from split fixture builds. + +Accept fixture directories as arguments, load each directory's +``.meta/index.json``, merge them via ``IndexFile.merge()``, and write +the result to the specified output path. +""" + +import sys +from pathlib import Path + +from execution_testing.fixtures.consume import IndexFile + + +def main() -> None: + """Entry point.""" + if len(sys.argv) < 3: + print( + "Usage: merge_index_files.py " + " [ ...]", + file=sys.stderr, + ) + sys.exit(1) + + output_path = Path(sys.argv[1]) + fixture_dirs = [Path(d) for d in sys.argv[2:]] + + indexes: list[IndexFile] = [] + for d in fixture_dirs: + index_path = d / ".meta" / "index.json" + if not index_path.exists(): + print(f"Skipping {d} (no .meta/index.json)") + continue + indexes.append(IndexFile.model_validate_json(index_path.read_text())) + + if not indexes: + print("No index files found, nothing to merge.") + sys.exit(0) + + merged = IndexFile.merge(indexes) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(merged.model_dump_json(indent=2)) + print(f"Merged {len(indexes)} index files ({merged.test_count} tests)") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/tests/test_release_scripts.py b/.github/scripts/tests/test_release_scripts.py new file mode 100644 index 00000000000..bd3acc17e8d --- /dev/null +++ b/.github/scripts/tests/test_release_scripts.py @@ -0,0 +1,286 @@ +""" +Test the CI release helper scripts. + +Each test invokes the script via ``uv run`` to validate the actual CLI +interface, matching how GitHub Actions calls them. +""" + +import json +import subprocess +import tarfile +from pathlib import Path + +SCRIPTS_DIR = Path(__file__).parent.parent +REPO_ROOT = SCRIPTS_DIR.parent.parent + +BUILD_MATRIX_SCRIPT = SCRIPTS_DIR / "generate_build_matrix.py" +TARBALL_SCRIPT = SCRIPTS_DIR / "create_release_tarball.py" +MERGE_INDEX_SCRIPT = SCRIPTS_DIR / "merge_index_files.py" + + +def run_script(script: Path, *args: str) -> subprocess.CompletedProcess: + """Run a uv inline-deps script and return the result.""" + return subprocess.run( + ["uv", "run", "-q", str(script), *args], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ) + + +def parse_matrix_output(stdout: str) -> dict[str, str]: + """Parse key=value output from generate_build_matrix.py.""" + return { + k: v + for line in stdout.strip().splitlines() + if "=" in line + for k, v in [line.split("=", 1)] + } + + +class TestGenerateBuildMatrix: + """Test generate_build_matrix.py.""" + + def test_split_feature_produces_entries_per_range(self): + """Verify a split feature expands into one entry per range.""" + result = run_script(BUILD_MATRIX_SCRIPT, "mainnet") + assert result.returncode == 0 + out = parse_matrix_output(result.stdout) + matrix = json.loads(out["build_matrix"]) + assert len(matrix) > 1 + assert out["feature_name"] == "mainnet" + assert out["combine_labels"] != "" + labels = [e["label"] for e in matrix] + assert all(lbl != "" for lbl in labels) + assert all(e["from_fork"] != "" for e in matrix) + assert all(e["until_fork"] != "" for e in matrix) + + def test_unsplit_feature_produces_single_entry(self): + """Verify a feature without fork-ranges produces one entry.""" + result = run_script(BUILD_MATRIX_SCRIPT, "benchmark") + assert result.returncode == 0 + out = parse_matrix_output(result.stdout) + matrix = json.loads(out["build_matrix"]) + assert len(matrix) == 1 + assert out["feature_name"] == "benchmark" + assert out["combine_labels"] == "" + assert matrix[0]["label"] == "" + assert matrix[0]["from_fork"] == "" + assert matrix[0]["until_fork"] == "" + + def test_feature_only_can_be_requested_explicitly(self): + """Verify feature_only entries work when named directly.""" + result = run_script(BUILD_MATRIX_SCRIPT, "bal") + assert result.returncode == 0 + out = parse_matrix_output(result.stdout) + matrix = json.loads(out["build_matrix"]) + assert len(matrix) == 1 + assert matrix[0]["feature"] == "bal" + assert out["combine_labels"] == "" + + def test_unknown_feature_fails(self): + """Verify error exit for unknown feature name.""" + result = run_script(BUILD_MATRIX_SCRIPT, "nonexistent") + assert result.returncode == 1 + assert "not found" in result.stderr + + def test_no_args_fails(self): + """Verify error exit when no arguments provided.""" + result = run_script(BUILD_MATRIX_SCRIPT) + assert result.returncode == 1 + assert "Usage" in result.stderr + + def test_output_is_valid_github_actions_format(self): + """Verify output lines are key=value for GITHUB_OUTPUT.""" + result = run_script(BUILD_MATRIX_SCRIPT, "mainnet") + assert result.returncode == 0 + lines = result.stdout.strip().splitlines() + assert len(lines) == 3 + assert lines[0].startswith("build_matrix=") + assert lines[1].startswith("feature_name=") + assert lines[2].startswith("combine_labels=") + + +class TestCreateReleaseTarball: + """Test create_release_tarball.py.""" + + def test_tarball_structure(self, tmp_path): + """Verify tarball has fixtures/ prefix and correct contents.""" + src = tmp_path / "fixtures" + (src / "blockchain_tests" / "for_cancun").mkdir(parents=True) + (src / "blockchain_tests_engine_x" / "pre_alloc").mkdir(parents=True) + (src / ".meta").mkdir() + + (src / "blockchain_tests" / "for_cancun" / "t.json").write_text("{}") + pre_alloc = src / "blockchain_tests_engine_x" / "pre_alloc" + (pre_alloc / "g.json").write_text("{}") + (src / ".meta" / "fixtures.ini").write_text("[meta]") + + out = tmp_path / "output.tar.gz" + result = run_script(TARBALL_SCRIPT, str(src), str(out)) + assert result.returncode == 0 + assert out.exists() + + with tarfile.open(out, "r:gz") as tar: + names = sorted(tar.getnames()) + + assert all(n.startswith("fixtures/") for n in names) + assert "fixtures/blockchain_tests/for_cancun/t.json" in names + assert "fixtures/blockchain_tests_engine_x/pre_alloc/g.json" in names + assert "fixtures/.meta/fixtures.ini" in names + + def test_excludes_non_fixture_files(self, tmp_path): + """Verify .log, .html, etc. are excluded from tarball.""" + src = tmp_path / "fixtures" + src.mkdir() + (src / "test.json").write_text("{}") + (src / "debug.log").write_text("log") + (src / "report.html").write_text("") + (src / "data.csv").write_text("a,b") + + out = tmp_path / "output.tar.gz" + result = run_script(TARBALL_SCRIPT, str(src), str(out)) + assert result.returncode == 0 + + with tarfile.open(out, "r:gz") as tar: + names = tar.getnames() + + assert "fixtures/test.json" in names + assert len(names) == 1 + + def test_nonexistent_dir_fails(self, tmp_path): + """Verify error for non-existent source directory.""" + result = run_script( + TARBALL_SCRIPT, + str(tmp_path / "nope"), + str(tmp_path / "out.tar.gz"), + ) + assert result.returncode == 1 + assert "not a directory" in result.stderr + + def test_no_args_fails(self): + """Verify error when no arguments provided.""" + result = run_script(TARBALL_SCRIPT) + assert result.returncode == 1 + assert "Usage" in result.stderr + + +def _run_merge_script( + *args: str, +) -> subprocess.CompletedProcess: + """Run merge_index_files.py via uv run python.""" + return subprocess.run( + ["uv", "run", "python", str(MERGE_INDEX_SCRIPT), *args], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ) + + +class TestMergeIndexFiles: + """Test merge_index_files.py.""" + + def _write_index(self, fixture_dir: Path, index_data: dict) -> None: + """Write a .meta/index.json file in the given directory.""" + meta = fixture_dir / ".meta" + meta.mkdir(parents=True, exist_ok=True) + (meta / "index.json").write_text(json.dumps(index_data)) + + def test_merges_two_index_files(self, tmp_path): + """Verify merging two fixture dirs produces a combined index.""" + dir_a = tmp_path / "fixtures__cancun" + dir_b = tmp_path / "fixtures__prague" + output = tmp_path / "combined" / ".meta" / "index.json" + + self._write_index( + dir_a, + { + "root_hash": None, + "created_at": "2026-01-01T00:00:00", + "test_count": 1, + "forks": ["Cancun"], + "fixture_formats": ["state_test"], + "test_cases": [ + { + "id": "test_a", + "json_path": "state_tests/for_cancun/t.json", + "fixture_hash": "0x" + "11" * 32, + "fork": "Cancun", + "format": "state_test", + } + ], + }, + ) + self._write_index( + dir_b, + { + "root_hash": None, + "created_at": "2026-01-01T00:00:00", + "test_count": 1, + "forks": ["Prague"], + "fixture_formats": ["blockchain_test"], + "test_cases": [ + { + "id": "test_b", + "json_path": "blockchain_tests/for_prague/t.json", + "fixture_hash": "0x" + "22" * 32, + "fork": "Prague", + "format": "blockchain_test", + } + ], + }, + ) + + result = _run_merge_script( + str(output), + str(dir_a), + str(dir_b), + ) + assert result.returncode == 0 + assert output.exists() + + merged = json.loads(output.read_text()) + assert merged["test_count"] == 2 + assert len(merged["test_cases"]) == 2 + assert merged["root_hash"] is not None + + def test_skips_dirs_without_index(self, tmp_path): + """Verify directories without .meta/index.json are skipped.""" + dir_a = tmp_path / "fixtures__cancun" + dir_a.mkdir() + dir_b = tmp_path / "fixtures__empty" + dir_b.mkdir() + output = tmp_path / "out.json" + + self._write_index( + dir_a, + { + "root_hash": None, + "created_at": "2026-01-01T00:00:00", + "test_count": 1, + "forks": ["Cancun"], + "fixture_formats": ["state_test"], + "test_cases": [ + { + "id": "test_a", + "json_path": "state_tests/t.json", + "fixture_hash": "0x" + "11" * 32, + "fork": "Cancun", + "format": "state_test", + } + ], + }, + ) + + result = _run_merge_script(str(output), str(dir_a), str(dir_b)) + assert result.returncode == 0 + assert output.exists() + + merged = json.loads(output.read_text()) + assert merged["test_count"] == 1 + + def test_no_args_fails(self): + """Verify error when no arguments provided.""" + result = _run_merge_script() + assert result.returncode == 1 + assert "Usage" in result.stderr diff --git a/.github/workflows/release_fixture_feature.yaml b/.github/workflows/release_fixture_feature.yaml index ba1ab399561..2f333ed6ff9 100644 --- a/.github/workflows/release_fixture_feature.yaml +++ b/.github/workflows/release_fixture_feature.yaml @@ -1,4 +1,4 @@ -name: Create Fixture Feature Release +name: Create Fixture Release on: push: @@ -7,32 +7,35 @@ on: workflow_dispatch: jobs: - feature-names: + setup: runs-on: ubuntu-latest outputs: - names: ${{ steps.feature-name.outputs.names }} + build_matrix: ${{ steps.matrix.outputs.build_matrix }} + feature_name: ${{ steps.matrix.outputs.feature_name }} + combine_labels: ${{ steps.matrix.outputs.combine_labels }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: false + - uses: ./.github/actions/setup-uv - - name: Get feature names - id: feature-name + - name: Generate build matrix + id: matrix shell: bash run: | FEATURE_PREFIX="${GITHUB_REF_NAME//@*/}" FEATURE_NAME="${FEATURE_PREFIX#tests-}" - names=$(grep -Po "^${FEATURE_NAME}(?=:)" .github/configs/feature.yaml | jq --raw-input . | jq -c --slurp .) - echo "names=${names}" - echo "names=${names}" >> "$GITHUB_OUTPUT" + uv run -q .github/scripts/generate_build_matrix.py "$FEATURE_NAME" >> "$GITHUB_OUTPUT" build: - needs: feature-names + name: fill (${{ matrix.label || matrix.feature }}) + needs: setup runs-on: [self-hosted-ghr, size-gigachungus-x64] timeout-minutes: 1440 strategy: + fail-fast: true matrix: - feature: ${{ fromJSON(needs.feature-names.outputs.names) }} + include: ${{ fromJson(needs.setup.outputs.build_matrix) }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -61,11 +64,67 @@ jobs: - uses: ./.github/actions/build-fixtures with: release_name: ${{ matrix.feature }} + from_fork: ${{ matrix.from_fork }} + until_fork: ${{ matrix.until_fork }} + split_label: ${{ matrix.label }} + + combine: + name: combine (${{ needs.setup.outputs.feature_name }}) + needs: [setup, build] + if: needs.setup.outputs.combine_labels != '' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: false + - uses: ./.github/actions/setup-uv + - name: Install pigz + run: sudo apt-get install -y pigz + - name: Download and merge split artifacts + shell: bash + run: | + mkdir -p combined + for label in ${{ needs.setup.outputs.combine_labels }}; do + echo "Downloading: fixtures__${label}" + if gh run download ${{ github.run_id }} -n "fixtures__${label}" --dir "split_artifacts/fixtures__${label}"; then + cp -r "split_artifacts/fixtures__${label}"/* combined/ + else + echo "No artifact for ${label} (no tests collected, skipping)" + fi + done + echo "Combined directory contents:" + find combined -maxdepth 3 -type d | head -30 || true + env: + GH_TOKEN: ${{ github.token }} + - name: Merge split index files + shell: bash + run: | + SPLIT_DIRS=() + for label in ${{ needs.setup.outputs.combine_labels }}; do + dir="split_artifacts/fixtures__${label}" + if [ -d "$dir" ]; then + SPLIT_DIRS+=("$dir") + fi + done + if [ ${#SPLIT_DIRS[@]} -gt 0 ]; then + uv run python .github/scripts/merge_index_files.py combined/.meta/index.json "${SPLIT_DIRS[@]}" + fi + - name: Free disk space + run: rm -rf split_artifacts/ + - name: Create release tarball + shell: bash + run: | + uv run -q .github/scripts/create_release_tarball.py combined fixtures_${{ needs.setup.outputs.feature_name }}.tar.gz + - name: Upload combined fixture tarball + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: fixtures_${{ needs.setup.outputs.feature_name }} + path: fixtures_${{ needs.setup.outputs.feature_name }}.tar.gz release: runs-on: ubuntu-latest - needs: build - if: startsWith(github.ref, 'refs/tags/tests-') + needs: [setup, build, combine] + if: always() && needs.build.result == 'success' && (needs.combine.result == 'success' || needs.combine.result == 'skipped') && startsWith(github.ref, 'refs/tags/tests-') permissions: contents: write steps: @@ -74,15 +133,17 @@ jobs: submodules: false fetch-depth: 0 - - name: Download all artifacts + - name: Download release artifacts + shell: bash run: | - gh run download ${{ github.run_id }} --dir ./artifacts + gh run download ${{ github.run_id }} -p "fixtures_*" --dir ./artifacts + rm -rf ./artifacts/fixtures__*/ + gh run download ${{ github.run_id }} -p "benchmark_genesis_*" --dir ./artifacts || true env: GH_TOKEN: ${{ github.token }} - - name: Draft pre-release on EELS (canonical) + - name: Draft release on EELS (canonical) run: | - # Find previous tag scoped to this feature (e.g., tests-bal@v*) FEATURE_PREFIX="${TAG_NAME%%@*}" PREV_TAG=$( git tag --list "${FEATURE_PREFIX}@v*" --sort=-v:refname \ @@ -90,24 +151,29 @@ jobs: | head -n 1 \ || true ) - NOTES_ARGS=() + RELEASE_ARGS=(--draft --generate-notes) + if [ "$FEATURE_NAME" != "mainnet" ]; then + RELEASE_ARGS+=(--prerelease) + fi if [ -n "$PREV_TAG" ]; then - NOTES_ARGS=(--notes-start-tag "$PREV_TAG") + RELEASE_ARGS+=(--notes-start-tag "$PREV_TAG") fi - gh release create "$TAG_NAME" --draft --prerelease --generate-notes "${NOTES_ARGS[@]}" ./artifacts/**/* + gh release create "$TAG_NAME" "${RELEASE_ARGS[@]}" ./artifacts/**/*.tar.gz env: TAG_NAME: ${{ github.ref_name }} + FEATURE_NAME: ${{ needs.setup.outputs.feature_name }} GH_TOKEN: ${{ github.token }} - - name: Draft pre-release on EEST (mirror) + - name: Draft release on EEST (mirror) run: | EEST_TAG="${TAG_NAME#tests-}" - gh release create "$EEST_TAG" \ - --repo ethereum/execution-spec-tests \ - --draft \ - --prerelease \ - --notes "This release is mirrored from [ethereum/execution-specs ${TAG_NAME}](https://github.com/ethereum/execution-specs/releases/tag/${TAG_NAME})." \ - ./artifacts/**/* + RELEASE_ARGS=(--repo ethereum/execution-spec-tests --draft) + if [ "$FEATURE_NAME" != "mainnet" ]; then + RELEASE_ARGS+=(--prerelease) + fi + RELEASE_ARGS+=(--notes "This release is mirrored from [ethereum/execution-specs ${TAG_NAME}](https://github.com/ethereum/execution-specs/releases/tag/${TAG_NAME}).") + gh release create "$EEST_TAG" "${RELEASE_ARGS[@]}" ./artifacts/**/*.tar.gz env: TAG_NAME: ${{ github.ref_name }} + FEATURE_NAME: ${{ needs.setup.outputs.feature_name }} GH_TOKEN: ${{ secrets.EEST_RELEASE_TOKEN }} diff --git a/.github/workflows/release_fixture_full.yaml b/.github/workflows/release_fixture_full.yaml deleted file mode 100644 index 48a5a2319b0..00000000000 --- a/.github/workflows/release_fixture_full.yaml +++ /dev/null @@ -1,109 +0,0 @@ -name: Create Fixture Full Release - -on: - push: - tags: - - "tests-v[0-9]+.[0-9]+.[0-9]+*" - workflow_dispatch: - -jobs: - features: - runs-on: ubuntu-latest - outputs: - features: ${{ steps.parse.outputs.features }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Get names from .github/configs/feature.yaml - id: parse - shell: bash - run: | - # Get all features without `feature_only: true` - grep -Po "^[0-9a-zA-Z_\-]+" ./.github/configs/feature.yaml | \ - while read -r feature; do - if ! awk "/^$feature:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag && /feature_only:.*true/{exit 1}" \ - ./.github/configs/feature.yaml; then - continue - fi - echo "$feature" - done | jq -R . | jq -cs . > features.json - echo "features=$(cat features.json)" >> "$GITHUB_OUTPUT" - - build: - needs: features - runs-on: [self-hosted-ghr, size-gigachungus-x64] - timeout-minutes: 1440 - strategy: - matrix: - name: ${{ fromJson(needs.features.outputs.features) }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: true - - name: Fetch lllc - uses: ./.github/actions/fetch-binary - with: - version: v1.0.0 - repo_owner: felix314159 - repo_name: lllc-custom - remote_name: lllc - binary_name: lllc - expected_sha256: 865a0d5379acb3b5471337b5dcf686a2dd71587c6b65b9da6c963de627e0b300 - - name: Fetch Solidity - uses: ./.github/actions/fetch-binary - with: - version: v0.8.24 - repo_owner: ethereum - repo_name: solidity - remote_name: solc-static-linux - binary_name: solc - expected_sha256: fb03a29a517452b9f12bcf459ef37d0a543765bb3bbc911e70a87d6a37c30d5f - - uses: ./.github/actions/build-fixtures - with: - release_name: ${{ matrix.name }} - - release: - runs-on: ubuntu-latest - needs: build - if: startsWith(github.ref, 'refs/tags/tests-') - permissions: - contents: write - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: false - fetch-depth: 0 - - - name: Download all artifacts - run: | - gh run download ${{ github.run_id }} --dir ./artifacts - env: - GH_TOKEN: ${{ github.token }} - - - name: Draft release on EELS (canonical) - run: | - PREV_TAG=$( - git tag --list 'tests-v[0-9]*' --sort=-v:refname \ - | grep -v "^${TAG_NAME}$" \ - | head -n 1 \ - || true - ) - NOTES_ARGS=() - if [ -n "$PREV_TAG" ]; then - NOTES_ARGS=(--notes-start-tag "$PREV_TAG") - fi - gh release create "$TAG_NAME" --draft --generate-notes "${NOTES_ARGS[@]}" ./artifacts/**/* - env: - TAG_NAME: ${{ github.ref_name }} - GH_TOKEN: ${{ github.token }} - - - name: Draft release on EEST (mirror) - run: | - EEST_TAG="${TAG_NAME#tests-}" - gh release create "$EEST_TAG" \ - --repo ethereum/execution-spec-tests \ - --draft \ - --notes "This release is mirrored from [ethereum/execution-specs ${TAG_NAME}](https://github.com/ethereum/execution-specs/releases/tag/${TAG_NAME})." \ - ./artifacts/**/* - env: - TAG_NAME: ${{ github.ref_name }} - GH_TOKEN: ${{ secrets.EEST_RELEASE_TOKEN }} diff --git a/Justfile b/Justfile index 1567a516e73..32d845d3536 100644 --- a/Justfile +++ b/Justfile @@ -196,6 +196,11 @@ test-tests-bench *args: "$@" \ packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_benchmarking.py +# Run CI release script integration tests +[group('unit tests')] +test-ci-scripts *args: + uv run pytest "$@" .github/scripts/tests/ + # --- Benchmarks --- # Fill benchmark tests with --gas-benchmark-values diff --git a/packages/testing/src/execution_testing/__init__.py b/packages/testing/src/execution_testing/__init__.py index e082013f43c..562fcfeea83 100644 --- a/packages/testing/src/execution_testing/__init__.py +++ b/packages/testing/src/execution_testing/__init__.py @@ -42,6 +42,7 @@ BlockchainTest, BlockchainTestFiller, Header, + OpcodeTarget, StateTest, StateTestFiller, TransactionTest, @@ -174,6 +175,7 @@ "Macros", "MemoryVariable", "NetworkWrappedTransaction", + "OpcodeTarget", "Op", "Opcode", "OpcodeCallArg", diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_output_directory.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_output_directory.py index 001f489b606..f818460f7ec 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_output_directory.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_output_directory.py @@ -258,6 +258,51 @@ def test_fill_to_tarball_directory( ) +def test_fill_single_fork_range( + pytester: pytest.Pytester, + minimal_test_path: Path, + tmp_path_factory: TempPathFactory, +) -> None: + """ + Test ``--from=FORK --until=FORK`` produces fixtures for one fork. + + Verify that setting both flags to the same fork (the pattern used by + split release runners) works for directory and tarball output. + """ + pytester.copy_example( + name="src/execution_testing/cli/pytest_commands/" + "pytest_ini_files/pytest-fill.ini" + ) + common_args = [ + "-c", + "pytest-fill.ini", + "-m", + "(not blockchain_test_engine) and (not eip_version_check)", + "--from=Cancun", + "--until=Cancun", + str(minimal_test_path), + ] + + # Directory output + dir_output = tmp_path_factory.mktemp("single_fork_dir") + result = pytester.runpytest(*common_args, f"--output={dir_output}") + assert result.ret == 0, f"Fill (dir) failed:\n{result.outlines}" + assert any(dir_output.glob("state_tests/**/*.json")), ( + "No state_test fixtures created (dir)" + ) + + # Tarball output + tarball_dir = tmp_path_factory.mktemp("single_fork_tar") + tarball_path = tarball_dir / "fixtures.tar.gz" + result = pytester.runpytest(*common_args, f"--output={tarball_path}") + assert result.ret == 0, f"Fill (tarball) failed:\n{result.outlines}" + assert tarball_path.exists(), "Tarball was not created" + extracted = tarball_dir / "fixtures" + assert any(extracted.glob("state_tests/**/*.json")), ( + "No state_test fixtures created (tarball)" + ) + + # New tests for the is_master functionality def test_create_directories_skips_when_not_master() -> None: """ diff --git a/packages/testing/src/execution_testing/cli/tests/test_hasher.py b/packages/testing/src/execution_testing/cli/tests/test_hasher.py index bd5c9357808..1c2c2fb3319 100644 --- a/packages/testing/src/execution_testing/cli/tests/test_hasher.py +++ b/packages/testing/src/execution_testing/cli/tests/test_hasher.py @@ -767,3 +767,99 @@ def test_merge_raises_when_no_partial_files(self) -> None: with pytest.raises(Exception, match="No partial indexes found"): merge_partial_indexes(output_dir, quiet_mode=True) + + +class TestIndexFileMerge: + """Test IndexFile.merge() for combining split fork-range indexes.""" + + def test_merge_combines_test_cases_and_metadata(self) -> None: + """Verify merge concatenates entries and unions forks/formats.""" + import datetime + + idx_a = IndexFile( + root_hash=None, + created_at=datetime.datetime(2026, 1, 1), + test_count=1, + forks=["Cancun"], + fixture_formats=["state_test"], + test_cases=[ + _make_entry( + "test_a", + "state_tests/for_cancun/t.json", + HASH_1, + fork="Cancun", + fmt="state_test", + ), + ], + ) + idx_b = IndexFile( + root_hash=None, + created_at=datetime.datetime(2026, 1, 2), + test_count=1, + forks=["Prague"], + fixture_formats=["blockchain_test"], + test_cases=[ + _make_entry( + "test_b", + "blockchain_tests/for_prague/t.json", + HASH_2, + fork="Prague", + fmt="blockchain_test", + ), + ], + ) + + merged = IndexFile.merge([idx_a, idx_b]) + + assert merged.test_count == 2 + assert len(merged.test_cases) == 2 + assert merged.forks is not None + assert set(f.name() for f in merged.forks) == {"Cancun", "Prague"} + assert merged.fixture_formats is not None + assert set(merged.fixture_formats) == { + "state_test", + "blockchain_test", + } + assert merged.root_hash is not None + + def test_merge_root_hash_matches_from_index_entries(self) -> None: + """Verify merged root_hash matches independent computation.""" + import datetime + + cases = [ + _make_entry( + "test_a", + "state_tests/for_cancun/t.json", + HASH_1, + fork="Cancun", + fmt="state_test", + ), + _make_entry( + "test_b", + "blockchain_tests/for_prague/t.json", + HASH_2, + fork="Prague", + fmt="blockchain_test", + ), + ] + + idx_a = IndexFile( + root_hash=None, + created_at=datetime.datetime(2026, 1, 1), + test_count=1, + forks=["Cancun"], + fixture_formats=["state_test"], + test_cases=cases[:1], + ) + idx_b = IndexFile( + root_hash=None, + created_at=datetime.datetime(2026, 1, 1), + test_count=1, + forks=["Prague"], + fixture_formats=["blockchain_test"], + test_cases=cases[1:], + ) + + merged = IndexFile.merge([idx_a, idx_b]) + expected_hash = HashableItem.from_index_entries(cases).hash() + assert merged.root_hash == HexNumber(expected_hash) diff --git a/packages/testing/src/execution_testing/fixtures/consume.py b/packages/testing/src/execution_testing/fixtures/consume.py index f7f36b617ad..499be630dd7 100644 --- a/packages/testing/src/execution_testing/fixtures/consume.py +++ b/packages/testing/src/execution_testing/fixtures/consume.py @@ -91,6 +91,36 @@ class IndexFile(BaseModel): fixture_formats: Optional[List[str]] = [] test_cases: List[TestCaseIndexFile] + @classmethod + def merge(cls, indexes: List["IndexFile"]) -> "IndexFile": + """ + Merge multiple index files into one. + + Concatenate test cases, union forks and fixture formats, and + recompute the root hash from the merged entries. + """ + from execution_testing.cli.hasher import HashableItem + + all_cases: List[TestCaseIndexFile] = [] + all_forks: set[Fork] = set() + all_formats: set[str] = set() + + for idx in indexes: + all_cases.extend(idx.test_cases) + all_forks.update(idx.forks or []) + all_formats.update(idx.fixture_formats or []) + + root_hash = HashableItem.from_index_entries(all_cases).hash() + + return cls( + root_hash=HexNumber(root_hash), + created_at=datetime.datetime.now(datetime.timezone.utc), + test_count=len(all_cases), + forks=sorted(all_forks, key=lambda f: f.name()), + fixture_formats=sorted(all_formats), + test_cases=all_cases, + ) + class TestCases(RootModel): """Root model defining a list test cases used in consume commands.""" diff --git a/packages/testing/src/execution_testing/specs/__init__.py b/packages/testing/src/execution_testing/specs/__init__.py index 1194e165135..e850fe4cb23 100644 --- a/packages/testing/src/execution_testing/specs/__init__.py +++ b/packages/testing/src/execution_testing/specs/__init__.py @@ -2,7 +2,12 @@ from .base import BaseTest, TestSpec from .base_static import BaseStaticTest -from .benchmark import BenchmarkTest, BenchmarkTestFiller, BenchmarkTestSpec +from .benchmark import ( + BenchmarkTest, + BenchmarkTestFiller, + BenchmarkTestSpec, + OpcodeTarget, +) from .blobs import BlobsTest, BlobsTestFiller, BlobsTestSpec from .blockchain import ( Block, @@ -35,6 +40,7 @@ "BlockchainTestSpec", "Block", "Header", + "OpcodeTarget", "StateStaticTest", "StateTest", "StateTestFiller", diff --git a/packages/testing/src/execution_testing/specs/benchmark.py b/packages/testing/src/execution_testing/specs/benchmark.py index 81ec46dcb90..174ae96bf73 100644 --- a/packages/testing/src/execution_testing/specs/benchmark.py +++ b/packages/testing/src/execution_testing/specs/benchmark.py @@ -45,6 +45,24 @@ from .blockchain import Block, BlockchainTest +@dataclass(frozen=True) +class OpcodeTarget: + """ + Map a display name to an underlying opcode for count validation. + + Use when the fixture metadata should show a descriptive label (e.g. a + precompile name) while opcode-count validation targets the real EVM + opcode that gets executed (e.g. STATICCALL). + """ + + name: str + opcode: Op + + def __str__(self) -> str: + """Return the display name.""" + return self.name + + @dataclass(kw_only=True) class BenchmarkCodeGenerator(ABC): """Abstract base class for generating benchmark bytecode.""" @@ -287,7 +305,7 @@ class BenchmarkTest(BaseTest): default_factory=lambda: int(Environment().gas_limit) ) fixed_opcode_count: float | None = None - target_opcode: Op | None = None + target_opcode: Op | OpcodeTarget | None = None code_generator: BenchmarkCodeGenerator | None = None # By default, benchmark tests require neither of these include_full_post_state_in_output: bool = False @@ -523,7 +541,12 @@ def _verify_target_opcode_count( # fixed_opcode_count is in thousands units expected = self.fixed_opcode_count * 1000 - actual = opcode_count.root.get(self.target_opcode, 0) + count_opcode = ( + self.target_opcode.opcode + if isinstance(self.target_opcode, OpcodeTarget) + else self.target_opcode + ) + actual = opcode_count.root.get(count_opcode, 0) tolerance = expected * 0.05 # 5% tolerance if abs(actual - expected) > tolerance: diff --git a/tests/benchmark/compute/helpers.py b/tests/benchmark/compute/helpers.py index 13be98d788d..a05203445a9 100644 --- a/tests/benchmark/compute/helpers.py +++ b/tests/benchmark/compute/helpers.py @@ -16,6 +16,7 @@ Initcode, IteratingBytecode, Op, + OpcodeTarget, TransactionWithCost, While, compute_create2_address, @@ -27,6 +28,30 @@ FieldElement, ) + +class Precompile: + """Target opcode labels for precompile benchmarks.""" + + ECRECOVER = OpcodeTarget("ECRECOVER", Op.STATICCALL) + SHA256 = OpcodeTarget("SHA2-256", Op.STATICCALL) + RIPEMD160 = OpcodeTarget("RIPEMD-160", Op.STATICCALL) + IDENTITY = OpcodeTarget("IDENTITY", Op.STATICCALL) + MODEXP = OpcodeTarget("MODEXP", Op.STATICCALL) + BN128_ADD = OpcodeTarget("BN128_ADD", Op.STATICCALL) + BN128_MUL = OpcodeTarget("BN128_MUL", Op.STATICCALL) + BN128_PAIRING = OpcodeTarget("BN128_PAIRING", Op.STATICCALL) + BLAKE2F = OpcodeTarget("BLAKE2F", Op.STATICCALL) + POINT_EVALUATION = OpcodeTarget("POINT_EVALUATION", Op.STATICCALL) + P256VERIFY = OpcodeTarget("P256VERIFY", Op.STATICCALL) + BLS12_G1ADD = OpcodeTarget("BLS12_G1ADD", Op.STATICCALL) + BLS12_G1MSM = OpcodeTarget("BLS12_G1MSM", Op.STATICCALL) + BLS12_G2ADD = OpcodeTarget("BLS12_G2ADD", Op.STATICCALL) + BLS12_G2MSM = OpcodeTarget("BLS12_G2MSM", Op.STATICCALL) + BLS12_PAIRING = OpcodeTarget("BLS12_PAIRING", Op.STATICCALL) + BLS12_MAP_FP_TO_G1 = OpcodeTarget("BLS12_MAP_FP_TO_G1", Op.STATICCALL) + BLS12_MAP_FP2_TO_G2 = OpcodeTarget("BLS12_MAP_FP2_TO_G2", Op.STATICCALL) + + DEFAULT_BINOP_ARGS = ( 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F, 0x73EDA753299D7D483339D80809A1D80553BDA402FFFE5BFEFFFFFFFF00000001, diff --git a/tests/benchmark/compute/precompile/test_alt_bn128.py b/tests/benchmark/compute/precompile/test_alt_bn128.py index 112bfcbe851..6466f554e7e 100644 --- a/tests/benchmark/compute/precompile/test_alt_bn128.py +++ b/tests/benchmark/compute/precompile/test_alt_bn128.py @@ -13,16 +13,17 @@ Fork, JumpLoopGenerator, Op, + OpcodeTarget, Transaction, While, ) from py_ecc.bn128 import G1, G2, multiply -from ..helpers import concatenate_parameters +from ..helpers import Precompile, concatenate_parameters @pytest.mark.parametrize( - "precompile_address,calldata", + "precompile_address,calldata,target", [ pytest.param( 0x06, @@ -34,6 +35,7 @@ "06614E20C147E940F2D70DA3F74C9A17DF361706A4485C742BD6788478FA17D7", ] ), + Precompile.BN128_ADD, id="bn128_add", marks=pytest.mark.repricing, ), @@ -49,6 +51,7 @@ "0000000000000000000000000000000000000000000000000000000000000000", ] ), + Precompile.BN128_ADD, id="bn128_add_infinities", marks=pytest.mark.repricing, ), @@ -64,6 +67,7 @@ "0000000000000000000000000000000000000000000000000000000000000002", ] ), + Precompile.BN128_ADD, id="bn128_add_1_2", ), pytest.param( @@ -75,6 +79,7 @@ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", ] ), + Precompile.BN128_MUL, id="bn128_mul", ), # Ported from @@ -88,6 +93,7 @@ "0000000000000000000000000000000000000000000000000000000000000002", ] ), + Precompile.BN128_MUL, id="bn128_mul_infinities_2_scalar", ), # Ported from @@ -101,6 +107,7 @@ "25f8c89ea3437f44f8fc8b6bfbb6312074dc6f983809a5e809ff4e1d076dd585", ] ), + Precompile.BN128_MUL, id="bn128_mul_infinities_32_byte_scalar", marks=pytest.mark.repricing, ), @@ -115,6 +122,7 @@ "0000000000000000000000000000000000000000000000000000000000000002", ] ), + Precompile.BN128_MUL, id="bn128_mul_1_2_2_scalar", ), # Ported from @@ -128,6 +136,7 @@ "25f8c89ea3437f44f8fc8b6bfbb6312074dc6f983809a5e809ff4e1d076dd585", ] ), + Precompile.BN128_MUL, id="bn128_mul_1_2_32_byte_scalar", ), # Ported from @@ -141,6 +150,7 @@ "0000000000000000000000000000000000000000000000000000000000000002", ] ), + Precompile.BN128_MUL, id="bn128_mul_32_byte_coord_and_2_scalar", marks=pytest.mark.repricing, ), @@ -155,6 +165,7 @@ "25f8c89ea3437f44f8fc8b6bfbb6312074dc6f983809a5e809ff4e1d076dd585", ] ), + Precompile.BN128_MUL, id="bn128_mul_32_byte_coord_and_scalar", marks=pytest.mark.repricing, ), @@ -178,6 +189,7 @@ "12C85EA5DB8C6DEB4AAB71808DCB408FE3D1E7690C43D37B4CE6CC0166FA7DAA", ] ), + Precompile.BN128_PAIRING, id="bn128_two_pairings", ), pytest.param( @@ -193,11 +205,17 @@ "120A2A4CF30C1BF9845F20C6FE39E07EA2CCE61F0C9BB048165FE5E4DE877550", ] ), + Precompile.BN128_PAIRING, id="bn128_one_pairing", ), # Ported from # https://github.com/NethermindEth/nethermind/blob/ceb8d57b8530ce8181d7427c115ca593386909d6/tools/EngineRequestsGenerator/TestCase.cs#L353 - pytest.param(0x08, [], id="ec_pairing_zero_input"), + pytest.param( + 0x08, + [], + Precompile.BN128_PAIRING, + id="ec_pairing_zero_input", + ), pytest.param( 0x08, concatenate_parameters( @@ -218,11 +236,13 @@ "3a8eb0b0996252cb548a4487da97b02422ebc0e834613f954de6c7e0afdc1fc0", ] ), + Precompile.BN128_PAIRING, id="ec_pairing_2_sets", ), pytest.param( 0x08, concatenate_parameters([""]), + Precompile.BN128_PAIRING, id="ec_pairing_1_pair", ), pytest.param( @@ -245,6 +265,7 @@ "12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", ] ), + Precompile.BN128_PAIRING, id="ec_pairing_2_pair", ), pytest.param( @@ -272,6 +293,7 @@ "12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", ] ), + Precompile.BN128_PAIRING, id="ec_pairing_3_pair", ), pytest.param( @@ -308,6 +330,7 @@ "2dc4cb08068b4aa5f14b7f1096ab35d5c13d78319ec7e66e9f67a1ff20cbbf03", ] ), + Precompile.BN128_PAIRING, id="ec_pairing_4_pair", ), pytest.param( @@ -351,6 +374,7 @@ "1ac5dac62d2332faa8069faca3b0d27fcdf95d8c8bafc9074ee72b5c1f33aa70", ] ), + Precompile.BN128_PAIRING, id="ec_pairing_5_pair", ), pytest.param( @@ -360,6 +384,7 @@ "0000000000000000000000000000000000000000000000000000000000000000", ] ), + Precompile.BN128_PAIRING, id="ec_pairing_1_pair_empty", ), ], @@ -368,6 +393,7 @@ def test_alt_bn128( benchmark_test: BenchmarkTestFiller, precompile_address: Address, calldata: bytes, + target: OpcodeTarget, ) -> None: """Benchmark ALT_BN128 precompile.""" attack_block = Op.POP( @@ -377,7 +403,7 @@ def test_alt_bn128( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=target, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, @@ -493,7 +519,7 @@ def test_bn128_pairings_amortized( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.BN128_PAIRING, code_generator=JumpLoopGenerator( setup=setup, attack_block=attack_block, @@ -520,7 +546,7 @@ def test_alt_bn128_benchmark( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.BN128_PAIRING, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, @@ -633,7 +659,7 @@ def test_ec_pairing( seed_offset += per_tx_variants benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.BN128_PAIRING, skip_gas_used_validation=True, blocks=[Block(txs=txs)], ) @@ -652,11 +678,21 @@ def _generate_g1_point(seed: int) -> Bytes: @pytest.mark.repricing @pytest.mark.parametrize( - "precompile_address,scalar", + "precompile_address,scalar,target", [ - pytest.param(0x06, None, id="ec_add"), - pytest.param(0x07, 2, id="ec_mul_small_scalar"), - pytest.param(0x07, 2**256 - 1, id="ec_mul_max_scalar"), + pytest.param(0x06, None, Precompile.BN128_ADD, id="ec_add"), + pytest.param( + 0x07, + 2, + Precompile.BN128_MUL, + id="ec_mul_small_scalar", + ), + pytest.param( + 0x07, + 2**256 - 1, + Precompile.BN128_MUL, + id="ec_mul_max_scalar", + ), ], ) def test_alt_bn128_uncachable( @@ -667,6 +703,7 @@ def test_alt_bn128_uncachable( tx_gas_limit: int, precompile_address: Address, scalar: int | None, + target: OpcodeTarget, ) -> None: """ Benchmark ecAdd/ecMul with unique input per call. @@ -722,6 +759,6 @@ def test_alt_bn128_uncachable( seed += 1 benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=target, blocks=[Block(txs=txs)], ) diff --git a/tests/benchmark/compute/precompile/test_blake2f.py b/tests/benchmark/compute/precompile/test_blake2f.py index db7e7ff2ae2..d88a2cc75d0 100644 --- a/tests/benchmark/compute/precompile/test_blake2f.py +++ b/tests/benchmark/compute/precompile/test_blake2f.py @@ -12,7 +12,7 @@ from tests.istanbul.eip152_blake2.common import Blake2bInput from tests.istanbul.eip152_blake2.spec import Spec as Blake2bSpec -from ..helpers import concatenate_parameters +from ..helpers import Precompile, concatenate_parameters @pytest.mark.parametrize( @@ -48,7 +48,7 @@ def test_blake2f( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.BLAKE2F, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, @@ -78,7 +78,7 @@ def test_blake2f_benchmark( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.BLAKE2F, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, diff --git a/tests/benchmark/compute/precompile/test_bls12_381.py b/tests/benchmark/compute/precompile/test_bls12_381.py index 7e59d955d20..b05ec9c76df 100644 --- a/tests/benchmark/compute/precompile/test_bls12_381.py +++ b/tests/benchmark/compute/precompile/test_bls12_381.py @@ -8,15 +8,16 @@ Fork, JumpLoopGenerator, Op, + OpcodeTarget, ) from tests.prague.eip2537_bls_12_381_precompiles import spec as bls12381_spec -from ..helpers import concatenate_parameters +from ..helpers import Precompile, concatenate_parameters @pytest.mark.parametrize( - "precompile_address,calldata", + "precompile_address,calldata,target", [ pytest.param( bls12381_spec.Spec.G1ADD, @@ -26,6 +27,7 @@ bls12381_spec.Spec.P1, ] ), + Precompile.BLS12_G1ADD, id="bls12_g1add", marks=pytest.mark.repricing, ), @@ -40,6 +42,7 @@ * (len(bls12381_spec.Spec.G1MSM_DISCOUNT_TABLE) - 1), ] ), + Precompile.BLS12_G1MSM, id="bls12_g1msm", ), pytest.param( @@ -50,6 +53,7 @@ bls12381_spec.Spec.P2, ] ), + Precompile.BLS12_G2ADD, id="bls12_g2add", marks=pytest.mark.repricing, ), @@ -68,6 +72,7 @@ * (len(bls12381_spec.Spec.G2MSM_DISCOUNT_TABLE) // 2), ] ), + Precompile.BLS12_G2MSM, id="bls12_g2msm", ), pytest.param( @@ -78,6 +83,7 @@ bls12381_spec.Spec.G2, ] ), + Precompile.BLS12_PAIRING, id="bls12_pairing_check", ), pytest.param( @@ -87,6 +93,7 @@ bls12381_spec.FP(bls12381_spec.Spec.P - 1), ] ), + Precompile.BLS12_MAP_FP_TO_G1, id="bls12_fp_to_g1", marks=pytest.mark.repricing, ), @@ -99,6 +106,7 @@ ), ] ), + Precompile.BLS12_MAP_FP2_TO_G2, id="bls12_fp_to_g2", marks=pytest.mark.repricing, ), @@ -109,6 +117,7 @@ def test_bls12_381( fork: Fork, precompile_address: Address, calldata: bytes, + target: OpcodeTarget, ) -> None: """Benchmark BLS12_381 precompile.""" if precompile_address not in fork.precompiles(): @@ -121,7 +130,7 @@ def test_bls12_381( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=target, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, @@ -155,7 +164,7 @@ def test_bls12_g1_msm( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.BLS12_G1MSM, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, @@ -206,7 +215,7 @@ def test_bls12_g2_msm( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.BLS12_G2MSM, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, @@ -239,7 +248,7 @@ def test_bls12_pairing( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.BLS12_PAIRING, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, diff --git a/tests/benchmark/compute/precompile/test_ecrecover.py b/tests/benchmark/compute/precompile/test_ecrecover.py index ab8b5cdbf28..7c0193746f9 100644 --- a/tests/benchmark/compute/precompile/test_ecrecover.py +++ b/tests/benchmark/compute/precompile/test_ecrecover.py @@ -9,7 +9,7 @@ Op, ) -from ..helpers import concatenate_parameters +from ..helpers import Precompile, concatenate_parameters @pytest.mark.repricing @@ -64,7 +64,7 @@ def test_ecrecover( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.ECRECOVER, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, diff --git a/tests/benchmark/compute/precompile/test_identity.py b/tests/benchmark/compute/precompile/test_identity.py index 2d447353950..56198ef1e14 100644 --- a/tests/benchmark/compute/precompile/test_identity.py +++ b/tests/benchmark/compute/precompile/test_identity.py @@ -8,7 +8,7 @@ Op, ) -from ..helpers import calculate_optimal_input_length +from ..helpers import Precompile, calculate_optimal_input_length def test_identity( @@ -36,7 +36,7 @@ def test_identity( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.IDENTITY, code_generator=JumpLoopGenerator( setup=Op.CODECOPY(0, 0, optimal_input_length), attack_block=attack_block, @@ -55,7 +55,7 @@ def test_identity_fixed_size( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.IDENTITY, code_generator=JumpLoopGenerator( setup=Op.CODECOPY(0, 0, size), attack_block=attack_block ), diff --git a/tests/benchmark/compute/precompile/test_modexp.py b/tests/benchmark/compute/precompile/test_modexp.py index 7de5ab8b9bc..4e62c72160c 100644 --- a/tests/benchmark/compute/precompile/test_modexp.py +++ b/tests/benchmark/compute/precompile/test_modexp.py @@ -10,6 +10,8 @@ from tests.byzantium.eip198_modexp_precompile.helpers import ModExpInput +from ..helpers import Precompile + def create_modexp_test_cases() -> list[ParameterSet]: """Create test cases for the MODEXP precompile.""" @@ -408,7 +410,7 @@ def test_modexp( ) ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.MODEXP, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, diff --git a/tests/benchmark/compute/precompile/test_p256verify.py b/tests/benchmark/compute/precompile/test_p256verify.py index 514f5989c55..04c997cf0e8 100644 --- a/tests/benchmark/compute/precompile/test_p256verify.py +++ b/tests/benchmark/compute/precompile/test_p256verify.py @@ -11,7 +11,7 @@ from tests.osaka.eip7951_p256verify_precompiles import spec as p256verify_spec -from ..helpers import concatenate_parameters +from ..helpers import Precompile, concatenate_parameters @pytest.mark.parametrize( @@ -81,7 +81,7 @@ def test_p256verify( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.P256VERIFY, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, diff --git a/tests/benchmark/compute/precompile/test_point_evaluation.py b/tests/benchmark/compute/precompile/test_point_evaluation.py index 09e84499feb..6d23f50402d 100644 --- a/tests/benchmark/compute/precompile/test_point_evaluation.py +++ b/tests/benchmark/compute/precompile/test_point_evaluation.py @@ -11,7 +11,7 @@ from tests.cancun.eip4844_blobs.spec import Spec as BlobsSpec -from ..helpers import concatenate_parameters +from ..helpers import Precompile, concatenate_parameters @pytest.mark.repricing @@ -50,7 +50,7 @@ def test_point_evaluation( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.POINT_EVALUATION, code_generator=JumpLoopGenerator( setup=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE), attack_block=attack_block, diff --git a/tests/benchmark/compute/precompile/test_ripemd160.py b/tests/benchmark/compute/precompile/test_ripemd160.py index 7f5601e8408..75aef42266a 100644 --- a/tests/benchmark/compute/precompile/test_ripemd160.py +++ b/tests/benchmark/compute/precompile/test_ripemd160.py @@ -8,7 +8,7 @@ Op, ) -from ..helpers import calculate_optimal_input_length +from ..helpers import Precompile, calculate_optimal_input_length def test_ripemd160( @@ -36,7 +36,7 @@ def test_ripemd160( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.RIPEMD160, code_generator=JumpLoopGenerator( setup=Op.CODECOPY(0, 0, optimal_input_length), attack_block=attack_block, @@ -55,7 +55,7 @@ def test_ripemd160_fixed_size( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.RIPEMD160, code_generator=JumpLoopGenerator( setup=Op.CODECOPY(0, 0, size), attack_block=attack_block ), diff --git a/tests/benchmark/compute/precompile/test_sha256.py b/tests/benchmark/compute/precompile/test_sha256.py index 7aa11e9c81e..def1d35bdae 100644 --- a/tests/benchmark/compute/precompile/test_sha256.py +++ b/tests/benchmark/compute/precompile/test_sha256.py @@ -8,7 +8,7 @@ Op, ) -from ..helpers import calculate_optimal_input_length +from ..helpers import Precompile, calculate_optimal_input_length def test_sha256( @@ -36,7 +36,7 @@ def test_sha256( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.SHA256, code_generator=JumpLoopGenerator( setup=Op.CODECOPY(0, 0, optimal_input_length), attack_block=attack_block, @@ -55,7 +55,7 @@ def test_sha256_fixed_size( ) benchmark_test( - target_opcode=Op.STATICCALL, + target_opcode=Precompile.SHA256, code_generator=JumpLoopGenerator( setup=Op.CODECOPY(0, 0, size), attack_block=attack_block ), diff --git a/tests/benchmark/stateful/bloatnet/test_single_opcode.py b/tests/benchmark/stateful/bloatnet/test_single_opcode.py index 060e1399fc4..1ca2b299754 100644 --- a/tests/benchmark/stateful/bloatnet/test_single_opcode.py +++ b/tests/benchmark/stateful/bloatnet/test_single_opcode.py @@ -35,6 +35,7 @@ keccak256, ) from execution_testing.base_types.base_types import Number +from execution_testing.rpc import EthRPC from tests.benchmark.stateful.helpers import ( ALLOWANCE_SELECTOR, @@ -42,8 +43,10 @@ BALANCEOF_SELECTOR, DECREMENT_COUNTER_CONDITION, MINT_SELECTOR, + STORAGE_BLOATED_EOAS, CacheStrategy, build_cache_strategy_blocks, + get_storage_bloated_eoa, ) REFERENCE_SPEC_GIT_PATH = "DUMMY/bloatnet.md" @@ -58,6 +61,204 @@ ) +def delegate_and_set_slot0( + pre: Alloc, + authority: EOA, + setter_address: Address, + slot_0_value: Hash, +) -> Transaction: + """ + Create a tx that delegates the authority to the setter and calls + it with slot_0_value as calldata, writing it into slot 0. + + The authority nonce is incremented in-place. + """ + tx = Transaction( + gas_limit=100_000, + to=authority, + value=0, + data=slot_0_value, + sender=pre.fund_eoa(), + authorization_list=[ + AuthorizationTuple( + chain_id=0, + address=setter_address, + nonce=authority.nonce, + signer=authority, + ), + ], + ) + authority.nonce = Number(authority.nonce + 1) + return tx + + +def run_bloated_eoa_benchmark( + *, + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + fork: Fork, + gas_benchmark_value: int, + tx_gas_limit: int, + token_name: str, + existing_slots: bool, + runtime_code: Bytecode, + eth_rpc: EthRPC | None = None, +) -> None: + """ + Run a bloated-EOA benchmark with the given runtime delegation code. + + Handles authority setup, slot 0 initialization, delegation to + runtime code, benchmark tx generation, and test invocation. + """ + authority = get_storage_bloated_eoa(token_name, eth_rpc=eth_rpc) + slot_0_value = Hash(1) if existing_slots else Hash(START_SLOT) + + setter_address = pre.deploy_contract(code=Op.SSTORE(0, Op.CALLDATALOAD(0))) + runtime_address = pre.deploy_contract(code=runtime_code) + + init_tx = delegate_and_set_slot0( + pre, authority, setter_address, slot_0_value + ) + runtime_tx = delegate_and_set_slot0( + pre, authority, runtime_address, Hash(0) + ) + + blocks: list[Block] = [Block(txs=[init_tx, runtime_tx])] + + gas_available = gas_benchmark_value + intrinsic_gas = fork.transaction_intrinsic_cost_calculator()() + sender = pre.fund_eoa() + + txs: list[Transaction] = [] + while gas_available >= intrinsic_gas: + tx_gas = min(gas_available, tx_gas_limit) + txs.append( + Transaction( + gas_limit=tx_gas, + to=authority, + sender=sender, + ) + ) + gas_available -= tx_gas + blocks.append(Block(txs=txs)) + + benchmark_test( + pre=pre, + blocks=blocks, + skip_gas_used_validation=True, + expected_receipt_status=True, + ) + + +@pytest.mark.repricing +@pytest.mark.parametrize("token_name", STORAGE_BLOATED_EOAS) +@pytest.mark.parametrize("existing_slots", [False, True]) +def test_sload_bloated( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + fork: Fork, + gas_benchmark_value: int, + tx_gas_limit: int, + token_name: str, + existing_slots: bool, + eth_rpc: EthRPC | None = None, +) -> None: + """ + Benchmark SLOAD opcodes targeting an EOA with storage bloated. + + The storage is assumed to be filled from 0-N linearly, where + each slot has the value of the key. If this is not the + storage layout of the target account, then the existing_slots + parameter will not be correct. + """ + runtime_code = ( + Op.SLOAD(Op.PUSH0) + + While( + body=(Op.DUP1 + Op.SLOAD + Op.POP + Op.PUSH1(1) + Op.ADD), + condition=Op.GT(Op.GAS, 0xFFFF), + ) + + Op.PUSH0 + + Op.SSTORE + ) + + run_bloated_eoa_benchmark( + benchmark_test=benchmark_test, + pre=pre, + fork=fork, + gas_benchmark_value=gas_benchmark_value, + tx_gas_limit=tx_gas_limit, + token_name=token_name, + existing_slots=existing_slots, + runtime_code=runtime_code, + eth_rpc=eth_rpc, + ) + + +@pytest.mark.repricing +@pytest.mark.parametrize("token_name", STORAGE_BLOATED_EOAS) +@pytest.mark.parametrize("write_new_value", [False, True]) +@pytest.mark.parametrize("existing_slots", [True, False]) +def test_sstore_bloated( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + fork: Fork, + gas_benchmark_value: int, + tx_gas_limit: int, + token_name: str, + write_new_value: bool, + existing_slots: bool, + eth_rpc: EthRPC | None = None, +) -> None: + """ + Benchmark SSTORE opcodes targeting an EOA with storage bloated. + + The storage is assumed to be filled from 0-N linearly, where + each slot has the value of the key. Except slot 0, this is the + pointer to the next free (empty) storage slot. + + For this test to work correctly under all parameters then above + has to be true. If this is not the case then some tests will not + test what they claim to do. For instance, for `write_new_value` + set to False we need to know the current value of the slots. + """ + stack_init = Op.DUP1 + ( + Op.PUSH1(1) + Op.ADD + Op.SWAP1 if write_new_value else Bytecode() + ) + + runtime_code = ( + Op.SLOAD(Op.PUSH0) + + stack_init + + While( + body=( + Op.DUP2 + + Op.DUP2 + + Op.SSTORE + + Op.PUSH1(1) + + Op.ADD + + Op.SWAP1 + + Op.PUSH1(1) + + Op.ADD + + Op.SWAP1 + ), + condition=Op.GT(Op.GAS, 0xFFFF), + ) + + Op.PUSH0 + + Op.SSTORE + ) + + run_bloated_eoa_benchmark( + benchmark_test=benchmark_test, + pre=pre, + fork=fork, + gas_benchmark_value=gas_benchmark_value, + tx_gas_limit=tx_gas_limit, + token_name=token_name, + existing_slots=existing_slots, + runtime_code=runtime_code, + eth_rpc=eth_rpc, + ) + + @pytest.mark.stub_parametrize( "erc20_stub", "test_sload_empty_erc20_balanceof_" ) @@ -1842,7 +2043,6 @@ def test_account_access( attack_call = Op.POP( opcode( address=address_retriever.address_op(), - gas=1, value=value_sent, # Gas accounting address_warm=access_warm, @@ -1855,8 +2055,6 @@ def test_account_access( attack_call = Op.POP( opcode( address=address_retriever.address_op(), - gas=1, - args_size=1024, # Gas accounting address_warm=access_warm, ) @@ -1873,6 +2071,7 @@ def test_account_access( loop_code = While( body=cache_op + attack_call + increment_op, + condition=Op.GT(Op.GAS, 0x9000) if value_sent > 0 else None, ) attack_code = IteratingBytecode( @@ -1884,9 +2083,13 @@ def test_account_access( ) # Calldata generator for each transaction of the iterating bytecode. + # Start from 1 to skip the Bittrex Controller's nonce=1 contract + # which has a non-payable fallback that reverts when receiving value. + calldata_offset = 1 if account_mode == AccountMode.EXISTING_CONTRACT else 0 + def calldata(iteration_count: int, start_iteration: int) -> bytes: del iteration_count - return Hash(start_iteration) + return Hash(start_iteration + calldata_offset) attack_address = pre.deploy_contract(code=attack_code, balance=10**21) @@ -1915,7 +2118,6 @@ def calldata(iteration_count: int, start_iteration: int) -> bytes: calldata=calldata, ) ) - total_gas_cost = sum(tx.gas_cost for tx in attack_txs) if cache_strategy == CacheStrategy.CACHE_PREVIOUS_BLOCK: with TestPhaseManager.setup(): @@ -1941,5 +2143,6 @@ def calldata(iteration_count: int, start_iteration: int) -> bytes: post=post, blocks=blocks, target_opcode=opcode, - expected_benchmark_gas_used=total_gas_cost, + skip_gas_used_validation=True, + expected_receipt_status=1, ) diff --git a/tests/benchmark/stateful/helpers.py b/tests/benchmark/stateful/helpers.py index 4ed97cf7ecc..6681c4920c1 100644 --- a/tests/benchmark/stateful/helpers.py +++ b/tests/benchmark/stateful/helpers.py @@ -4,6 +4,7 @@ from enum import Enum from execution_testing import ( + EOA, AccessList, Address, Alloc, @@ -13,6 +14,8 @@ Op, Transaction, ) +from execution_testing.base_types import Number +from execution_testing.rpc import EthRPC # ERC20 function selectors BALANCEOF_SELECTOR = 0x70A08231 # balanceOf(address) @@ -20,6 +23,35 @@ ALLOWANCE_SELECTOR = 0xDD62ED3E # allowance(address,address) MINT_SELECTOR = 0x40C10F19 # mint(address,uint256) +# Storage-bloated EOA private keys, keyed by bloat size identifier. +# Addresses derived via: keccak256(utf8ToBytes("stateBloaters{N}")) +_STORAGE_BLOATED_EOA_KEYS: dict[str, str] = { + "1GB": ( + "0xc618d7bcd54de2f0dcf86e4ced86ccf07926619a74ee10432c3d1c60743e3427" + ), + "10GB": ( + "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e" + ), + "20GB": ( + "0xc025d5a1aa0f5eee1f50687901c5dc9a8e97a2be91aa381e4c938dc309105059" + ), +} + +STORAGE_BLOATED_EOAS: list[str] = list(_STORAGE_BLOATED_EOA_KEYS.keys()) + + +def get_storage_bloated_eoa( + name: str, + eth_rpc: EthRPC | None = None, +) -> EOA: + """Return an EOA for a storage-bloated account with its on-chain nonce.""" + eoa = EOA(key=_STORAGE_BLOATED_EOA_KEYS[name]) + if eth_rpc is not None: + nonce = eth_rpc.get_transaction_count(Address(eoa)) + eoa.nonce = Number(nonce) + return eoa + + # Standard While-loop decrement-and-test condition. # # Expects the iteration counter on top of the stack: