From b4d418d8ce7b45fa45f36728fb747844c78818a9 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 15 Jan 2024 17:34:28 +0200 Subject: [PATCH 01/26] Remove some trailing whitespace --- specs/_features/eip7594/polynomial-commitments-sampling.md | 6 +++--- specs/deneb/polynomial-commitments.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index b356792e34..0c3ec8fb0d 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -242,7 +242,7 @@ def interpolate_polynomialcoeff(xs: Sequence[BLSFieldElement], ys: Sequence[BLSF summand, [(- int(weight_adjustment) * int(xs[j])) % BLS_MODULUS, weight_adjustment] ) r = add_polynomialcoeff(r, summand) - + return r ``` @@ -385,7 +385,7 @@ def compute_cells(blob: Blob) -> Vector[Cell, CELLS_PER_BLOB]: polynomial = blob_to_polynomial(blob) polynomial_coeff = polynomial_eval_to_coeff(polynomial) - extended_data = fft_field(polynomial_coeff + [0] * FIELD_ELEMENTS_PER_BLOB, + extended_data = fft_field(polynomial_coeff + [0] * FIELD_ELEMENTS_PER_BLOB, compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) extended_data_rbo = bit_reversal_permutation(extended_data) return [extended_data_rbo[i * FIELD_ELEMENTS_PER_CELL:(i + 1) * FIELD_ELEMENTS_PER_CELL] @@ -504,7 +504,7 @@ def recover_polynomial(cell_ids: Sequence[CellID], cells: Sequence[Cell]) -> Pol eval_shifted_extended_evaluation = fft_field(shifted_extended_evaluation, roots_of_unity_extended) eval_shifted_zero_poly = fft_field(shifted_zero_poly, roots_of_unity_extended) - + eval_shifted_reconstructed_poly = [ div(a, b) for a, b in zip(eval_shifted_extended_evaluation, eval_shifted_zero_poly) diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index d7a0bc5035..33945d249f 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -578,7 +578,7 @@ def verify_blob_kzg_proof_batch(blobs: Sequence[Blob], """ assert len(blobs) == len(commitments_bytes) == len(proofs_bytes) - + commitments, evaluation_challenges, ys, proofs = [], [], [], [] for blob, commitment_bytes, proof_bytes in zip(blobs, commitments_bytes, proofs_bytes): assert len(blob) == BYTES_PER_BLOB From a4331e7c6346edebdf88a0e202d23d031acc69b4 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 15 Jan 2024 18:27:55 +0200 Subject: [PATCH 02/26] Public methods must accept raw bytes --- .../polynomial-commitments-sampling.md | 50 ++++++++++++++----- .../test_polynomial_commitments.py | 25 +++++++--- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 0c3ec8fb0d..397661cd7f 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -12,6 +12,8 @@ - [Preset](#preset) - [Cells](#cells) - [Helper functions](#helper-functions) + - [BLS12-381 helpers](#bls12-381-helpers) + - [`bytes_to_cell`](#bytes_to_cell) - [Linear combinations](#linear-combinations) - [`g2_lincomb`](#g2_lincomb) - [FFTs](#ffts) @@ -81,6 +83,18 @@ Cells are the smallest unit of blob data that can come with their own KZG proofs ## Helper functions +### BLS12-381 helpers + +#### `bytes_to_cell` + +```python +def bytes_to_cell(cell_bytes: Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]) -> Cell: + """ + Convert untrusted bytes into a Cell. + """ + return [bytes_to_bls_field(element) for element in cell_bytes] +``` + ### Linear combinations #### `g2_lincomb` @@ -397,10 +411,10 @@ def compute_cells(blob: Blob) -> Vector[Cell, CELLS_PER_BLOB]: #### `verify_cell_proof` ```python -def verify_cell_proof(commitment: KZGCommitment, +def verify_cell_proof(commitment_bytes: Bytes48, cell_id: int, - cell: Cell, - proof: KZGProof) -> bool: + cell_bytes: Vector[Bytes32, FIELD_ELEMENTS_PER_CELL], + proof: Bytes48) -> bool: """ Check a cell proof @@ -408,17 +422,21 @@ def verify_cell_proof(commitment: KZGCommitment, """ coset = coset_for_cell(cell_id) - return verify_kzg_proof_multi_impl(commitment, coset, cell, proof) + return verify_kzg_proof_multi_impl( + bytes_to_kzg_commitment(commitment_bytes), + coset, + bytes_to_cell(cell_bytes), + bytes_to_kzg_proof(proof)) ``` #### `verify_cell_proof_batch` ```python -def verify_cell_proof_batch(row_commitments: Sequence[KZGCommitment], +def verify_cell_proof_batch(row_commitments_bytes: Sequence[Bytes48], row_ids: Sequence[int], column_ids: Sequence[int], - cells: Sequence[Cell], - proofs: Sequence[KZGProof]) -> bool: + cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]], + proofs_bytes: Sequence[Bytes48]) -> bool: """ Check multiple cell proofs. This function implements the naive algorithm of checking every cell individually; an efficient algorithm can be found here: @@ -430,10 +448,14 @@ def verify_cell_proof_batch(row_commitments: Sequence[KZGCommitment], Public method. """ - # Get commitments via row IDs - commitments = [row_commitments[row_id] for row_id in row_ids] - + commitments_bytes = [row_commitments_bytes[row_id] for row_id in row_ids] + + # Get objects from bytes + commitments = [bytes_to_kzg_commitment(commitment_bytes) for commitment_bytes in commitments_bytes] + cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] + proofs = [bytes_to_kzg_proof(proof_bytes) for proof_bytes in proofs_bytes] + return all( verify_kzg_proof_multi_impl(commitment, coset_for_cell(column_id), cell, proof) for commitment, column_id, cell, proof in zip(commitments, column_ids, cells, proofs) @@ -445,7 +467,8 @@ def verify_cell_proof_batch(row_commitments: Sequence[KZGCommitment], ### `recover_polynomial` ```python -def recover_polynomial(cell_ids: Sequence[CellID], cells: Sequence[Cell]) -> Polynomial: +def recover_polynomial(cell_ids: Sequence[CellID], + cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]]) -> Polynomial: """ Recovers a polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. @@ -455,7 +478,10 @@ def recover_polynomial(cell_ids: Sequence[CellID], cells: Sequence[Cell]) -> Pol Public method. """ - assert len(cell_ids) == len(cells) + assert len(cell_ids) == len(cells_bytes) + + cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] + assert len(cells) >= CELLS_PER_BLOB // 2 missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] roots_of_unity_reduced = compute_roots_of_unity(CELLS_PER_BLOB) diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py index d3e848772a..bea38d6e5b 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -10,6 +10,10 @@ from eth2spec.utils.bls import BLS_MODULUS +def field_element_bytes(x): + return int.to_bytes(x % BLS_MODULUS, 32, "big") + + @with_eip7594_and_later @spec_test @single_phase @@ -34,10 +38,13 @@ def test_verify_cell_proof(spec): blob = get_sample_blob(spec) commitment = spec.blob_to_kzg_commitment(blob) cells, proofs = spec.compute_cells_and_proofs(blob) + + cells_bytes = [[field_element_bytes(element) for element in cell] for cell in cells] + cell_id = 0 - assert spec.verify_cell_proof(commitment, cell_id, cells[cell_id], proofs[cell_id]) + assert spec.verify_cell_proof(commitment, cell_id, cells_bytes[cell_id], proofs[cell_id]) cell_id = 1 - assert spec.verify_cell_proof(commitment, cell_id, cells[cell_id], proofs[cell_id]) + assert spec.verify_cell_proof(commitment, cell_id, cells_bytes[cell_id], proofs[cell_id]) @with_eip7594_and_later @@ -48,12 +55,14 @@ def test_verify_cell_proof_batch(spec): commitment = spec.blob_to_kzg_commitment(blob) cells, proofs = spec.compute_cells_and_proofs(blob) + cells_bytes = [[field_element_bytes(element) for element in cell] for cell in cells] + assert spec.verify_cell_proof_batch( - row_commitments=[commitment], + row_commitments_bytes=[commitment], row_ids=[0], column_ids=[0, 1], - cells=cells[0:1], - proofs=proofs, + cells_bytes=cells_bytes[0:1], + proofs_bytes=proofs, ) @@ -73,10 +82,10 @@ def test_recover_polynomial(spec): # Extend data with Reed-Solomon and split the extended data in cells cells = spec.compute_cells(blob) + cells_bytes = [[field_element_bytes(element) for element in cell] for cell in cells] # Compute the cells we will be recovering from cell_ids = [] - known_cells = [] # First figure out just the indices of the cells for i in range(N_SAMPLES): j = rng.randint(0, spec.CELLS_PER_BLOB) @@ -84,10 +93,10 @@ def test_recover_polynomial(spec): j = rng.randint(0, spec.CELLS_PER_BLOB) cell_ids.append(j) # Now the cells themselves - known_cells = [cells[cell_id] for cell_id in cell_ids] + known_cells_bytes = [cells_bytes[cell_id] for cell_id in cell_ids] # Recover the data - recovered_data = spec.recover_polynomial(cell_ids, known_cells) + recovered_data = spec.recover_polynomial(cell_ids, known_cells_bytes) # Check that the original data match the non-extended portion of the recovered data assert original_polynomial == recovered_data[:len(recovered_data) // 2] From 96e41bca3015d4f4c8d8d136c3e217081df161fe Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Jan 2024 16:51:05 +0800 Subject: [PATCH 03/26] Optimization: reduce `len()` call --- specs/_features/eip7594/polynomial-commitments-sampling.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index b356792e34..601144abd8 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -156,7 +156,9 @@ def add_polynomialcoeff(a: PolynomialCoeff, b: PolynomialCoeff) -> PolynomialCoe Sum the coefficient form polynomials ``a`` and ``b``. """ a, b = (a, b) if len(a) >= len(b) else (b, a) - return [(a[i] + (b[i] if i < len(b) else 0)) % BLS_MODULUS for i in range(len(a))] + length_a = len(a) + length_b = len(b) + return [(a[i] + (b[i] if i < length_b else 0)) % BLS_MODULUS for i in range(length_a)] ``` #### `neg_polynomialcoeff` From 87f2ad268103832cfea4a1ccd2a0192b0c04608d Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 16 Jan 2024 16:58:14 +0200 Subject: [PATCH 04/26] Public methods: int -> uint64 --- .../_features/eip7594/polynomial-commitments-sampling.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 397661cd7f..9f4584d341 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -344,7 +344,7 @@ def verify_kzg_proof_multi_impl(commitment: KZGCommitment, #### `coset_for_cell` ```python -def coset_for_cell(cell_id: int) -> Cell: +def coset_for_cell(cell_id: CellID) -> Cell: """ Get the coset for a given ``cell_id`` """ @@ -412,7 +412,7 @@ def compute_cells(blob: Blob) -> Vector[Cell, CELLS_PER_BLOB]: ```python def verify_cell_proof(commitment_bytes: Bytes48, - cell_id: int, + cell_id: uint64, cell_bytes: Vector[Bytes32, FIELD_ELEMENTS_PER_CELL], proof: Bytes48) -> bool: """ @@ -433,8 +433,8 @@ def verify_cell_proof(commitment_bytes: Bytes48, ```python def verify_cell_proof_batch(row_commitments_bytes: Sequence[Bytes48], - row_ids: Sequence[int], - column_ids: Sequence[int], + row_ids: Sequence[uint64], + column_ids: Sequence[uint64], cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]], proofs_bytes: Sequence[Bytes48]) -> bool: """ From 66798602dd448383361b026798ca734119200255 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 17 Jan 2024 14:18:03 +0200 Subject: [PATCH 05/26] Minor fixes based on review --- specs/_features/eip7594/polynomial-commitments-sampling.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 9f4584d341..f369becf28 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -412,9 +412,9 @@ def compute_cells(blob: Blob) -> Vector[Cell, CELLS_PER_BLOB]: ```python def verify_cell_proof(commitment_bytes: Bytes48, - cell_id: uint64, + cell_id: CellID, cell_bytes: Vector[Bytes32, FIELD_ELEMENTS_PER_CELL], - proof: Bytes48) -> bool: + proof_bytes: Bytes48) -> bool: """ Check a cell proof @@ -426,7 +426,7 @@ def verify_cell_proof(commitment_bytes: Bytes48, bytes_to_kzg_commitment(commitment_bytes), coset, bytes_to_cell(cell_bytes), - bytes_to_kzg_proof(proof)) + bytes_to_kzg_proof(proof_bytes)) ``` #### `verify_cell_proof_batch` From 2000a4f30770747ca696d7e23e355256c8f675c7 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 17 Jan 2024 17:20:36 +0200 Subject: [PATCH 06/26] Improve test_verify_cell_proof_batch() --- .../eip7594/polynomial-commitments-sampling.md | 7 ++++++- .../test_polynomial_commitments.py | 11 ++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index f369becf28..e22b6bf9fd 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -438,7 +438,10 @@ def verify_cell_proof_batch(row_commitments_bytes: Sequence[Bytes48], cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]], proofs_bytes: Sequence[Bytes48]) -> bool: """ - Check multiple cell proofs. This function implements the naive algorithm of checking every cell + Verify a set of cells, given their corresponding proofs and their coordinates (row_id, column_id) in the blob + matrix. The list of all commitments is also provided in row_commitments_bytes. + + This function implements the naive algorithm of checking every cell individually; an efficient algorithm can be found here: https://ethresear.ch/t/a-universal-verification-equation-for-data-availability-sampling/13240 @@ -448,6 +451,8 @@ def verify_cell_proof_batch(row_commitments_bytes: Sequence[Bytes48], Public method. """ + assert len(cells_bytes) == len(proofs_bytes) == len(row_ids) == len(column_ids) + # Get commitments via row IDs commitments_bytes = [row_commitments_bytes[row_id] for row_id in row_ids] diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py index bea38d6e5b..0b60cfd287 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -54,15 +54,16 @@ def test_verify_cell_proof_batch(spec): blob = get_sample_blob(spec) commitment = spec.blob_to_kzg_commitment(blob) cells, proofs = spec.compute_cells_and_proofs(blob) - cells_bytes = [[field_element_bytes(element) for element in cell] for cell in cells] + assert len(cells) == len(proofs) + assert spec.verify_cell_proof_batch( row_commitments_bytes=[commitment], - row_ids=[0], - column_ids=[0, 1], - cells_bytes=cells_bytes[0:1], - proofs_bytes=proofs, + row_ids=[0, 0], + column_ids=[0, 4], + cells_bytes=[cells_bytes[0], cells_bytes[4]], + proofs_bytes=[proofs[0], proofs[4]], ) From 0db4580e815bf0e713cc4ca3a47f92018262673a Mon Sep 17 00:00:00 2001 From: Guruprasad Kamath Date: Tue, 23 Jan 2024 20:53:05 +0100 Subject: [PATCH 07/26] add py.typed file mypy When this is installed as a dependency in other projects, the py.typed file helps the other project to read the type hints from the consensus-specs --- py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 py.typed diff --git a/py.typed b/py.typed new file mode 100644 index 0000000000..e69de29bb2 From 8b3ac46a2dff4bddbefc4e31259ec7fc1b47eb19 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 31 Jan 2024 13:45:49 +0200 Subject: [PATCH 08/26] Add helper method: construct_vanishing_polynomial() --- .../polynomial-commitments-sampling.md | 62 ++++++++++++------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index d3a8cd23bd..590a4efec4 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -471,25 +471,13 @@ def verify_cell_proof_batch(row_commitments_bytes: Sequence[Bytes48], ## Reconstruction -### `recover_polynomial` +### `construct_vanishing_polynomial` ```python -def recover_polynomial(cell_ids: Sequence[CellID], - cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]]) -> Polynomial: - """ - Recovers a polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. - - This algorithm uses FFTs to recover cells faster than using Lagrange implementation. However, - a faster version thanks to Qi Zhou can be found here: - https://github.com/ethereum/research/blob/51b530a53bd4147d123ab3e390a9d08605c2cdb8/polynomial_reconstruction/polynomial_reconstruction_danksharding.py - - Public method. - """ - assert len(cell_ids) == len(cells_bytes) - - cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] - - assert len(cells) >= CELLS_PER_BLOB // 2 +def construct_vanishing_polynomial(cell_ids: Sequence[CellID], + cells: Sequence[Cell]) -> Tuple[ + Sequence[BLSFieldElement], + Sequence[BLSFieldElement]]: missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] roots_of_unity_reduced = compute_roots_of_unity(CELLS_PER_BLOB) short_zero_poly = vanishing_polynomialcoeff([ @@ -497,13 +485,13 @@ def recover_polynomial(cell_ids: Sequence[CellID], for cell_id in missing_cell_ids ]) - full_zero_poly = [] + zero_poly_coeff = [] for i in short_zero_poly: - full_zero_poly.append(i) - full_zero_poly.extend([0] * (FIELD_ELEMENTS_PER_CELL - 1)) - full_zero_poly = full_zero_poly + [0] * (2 * FIELD_ELEMENTS_PER_BLOB - len(full_zero_poly)) + zero_poly_coeff.append(i) + zero_poly_coeff.extend([0] * (FIELD_ELEMENTS_PER_CELL - 1)) + zero_poly_coeff = zero_poly_coeff + [0] * (2 * FIELD_ELEMENTS_PER_BLOB - len(zero_poly_coeff)) - zero_poly_eval = fft_field(full_zero_poly, + zero_poly_eval = fft_field(zero_poly_coeff, compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) zero_poly_eval_brp = bit_reversal_permutation(zero_poly_eval) for cell_id in missing_cell_ids: @@ -515,6 +503,34 @@ def recover_polynomial(cell_ids: Sequence[CellID], end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL assert all(a != 0 for a in zero_poly_eval_brp[start:end]) + return zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp +``` + +### `recover_shifted_data` + +### `recover_original_data` + +### `recover_polynomial` + +```python +def recover_polynomial(cell_ids: Sequence[CellID], + cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]]) -> Polynomial: + """ + Recovers a polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. + + This algorithm uses FFTs to recover cells faster than using Lagrange implementation. However, + a faster version thanks to Qi Zhou can be found here: + https://github.com/ethereum/research/blob/51b530a53bd4147d123ab3e390a9d08605c2cdb8/polynomial_reconstruction/polynomial_reconstruction_danksharding.py + + Public method. + """ + assert len(cell_ids) == len(cells_bytes) + + cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] + assert len(cells) >= CELLS_PER_BLOB // 2 + + zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(cell_ids, cells) + extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL @@ -533,7 +549,7 @@ def recover_polynomial(cell_ids: Sequence[CellID], shift_inv = div(BLSFieldElement(1), shift_factor) shifted_extended_evaluation = shift_polynomialcoeff(extended_evaluations_fft, shift_factor) - shifted_zero_poly = shift_polynomialcoeff(full_zero_poly, shift_factor) + shifted_zero_poly = shift_polynomialcoeff(zero_poly_coeff, shift_factor) eval_shifted_extended_evaluation = fft_field(shifted_extended_evaluation, roots_of_unity_extended) eval_shifted_zero_poly = fft_field(shifted_zero_poly, roots_of_unity_extended) From be5e5c5a75231cd6c0bd255470181bc30613d425 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 23 Jan 2024 18:22:05 +0200 Subject: [PATCH 09/26] Add helpers: recover_shifted_data() and recover_original_data() --- .../polynomial-commitments-sampling.md | 78 +++++++++++++------ 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 590a4efec4..be215ee049 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -42,6 +42,9 @@ - [`verify_cell_proof`](#verify_cell_proof) - [`verify_cell_proof_batch`](#verify_cell_proof_batch) - [Reconstruction](#reconstruction) + - [`construct_vanishing_polynomial`](#construct_vanishing_polynomial) + - [`recover_shifted_data`](#recover_shifted_data) + - [`recover_original_data`](#recover_original_data) - [`recover_polynomial`](#recover_polynomial) @@ -508,29 +511,15 @@ def construct_vanishing_polynomial(cell_ids: Sequence[CellID], ### `recover_shifted_data` -### `recover_original_data` - -### `recover_polynomial` - ```python -def recover_polynomial(cell_ids: Sequence[CellID], - cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]]) -> Polynomial: - """ - Recovers a polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. - - This algorithm uses FFTs to recover cells faster than using Lagrange implementation. However, - a faster version thanks to Qi Zhou can be found here: - https://github.com/ethereum/research/blob/51b530a53bd4147d123ab3e390a9d08605c2cdb8/polynomial_reconstruction/polynomial_reconstruction_danksharding.py - - Public method. - """ - assert len(cell_ids) == len(cells_bytes) - - cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] - assert len(cells) >= CELLS_PER_BLOB // 2 - - zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(cell_ids, cells) - +def recover_shifted_data(cell_ids: Sequence[CellID], + cells: Sequence[Cell], + zero_poly_eval: Sequence[BLSFieldElement], + zero_poly_coeff: Sequence[BLSFieldElement], + roots_of_unity_extended: Sequence[BLSFieldElement]) -> Tuple[ + Sequence[BLSFieldElement], + Sequence[BLSFieldElement], + BLSFieldElement]: extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL @@ -541,8 +530,6 @@ def recover_polynomial(cell_ids: Sequence[CellID], extended_evaluation_times_zero = [BLSFieldElement(int(a) * int(b) % BLS_MODULUS) for a, b in zip(zero_poly_eval, extended_evaluation)] - roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) - extended_evaluations_fft = fft_field(extended_evaluation_times_zero, roots_of_unity_extended, inv=True) shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) @@ -554,6 +541,16 @@ def recover_polynomial(cell_ids: Sequence[CellID], eval_shifted_extended_evaluation = fft_field(shifted_extended_evaluation, roots_of_unity_extended) eval_shifted_zero_poly = fft_field(shifted_zero_poly, roots_of_unity_extended) + return eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv +``` + +### `recover_original_data` + +```python +def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldElement], + eval_shifted_zero_poly: Sequence[BLSFieldElement], + shift_inv: BLSFieldElement, + roots_of_unity_extended: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]: eval_shifted_reconstructed_poly = [ div(a, b) for a, b in zip(eval_shifted_extended_evaluation, eval_shifted_zero_poly) @@ -565,6 +562,39 @@ def recover_polynomial(cell_ids: Sequence[CellID], reconstructed_data = bit_reversal_permutation(fft_field(reconstructed_poly, roots_of_unity_extended)) + return reconstructed_data +``` + +### `recover_polynomial` + +```python +def recover_polynomial(cell_ids: Sequence[CellID], + cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]]) -> Polynomial: + """ + Recovers a polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. + + This algorithm uses FFTs to recover cells faster than using Lagrange implementation. However, + a faster version thanks to Qi Zhou can be found here: + https://github.com/ethereum/research/blob/51b530a53bd4147d123ab3e390a9d08605c2cdb8/polynomial_reconstruction/polynomial_reconstruction_danksharding.py + + Public method. + """ + assert len(cell_ids) == len(cells_bytes) + + cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] + assert len(cells) >= CELLS_PER_BLOB // 2 + + roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) + + zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(cell_ids, cells) + + eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = \ + recover_shifted_data(cell_ids, cells, zero_poly_eval, zero_poly_coeff, roots_of_unity_extended) + + reconstructed_data = \ + recover_original_data(eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv, + roots_of_unity_extended) + for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL From 4d01d5043720ed9516186521fe1f8696ef809420 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 31 Jan 2024 13:45:04 +0200 Subject: [PATCH 10/26] Initial docs added to recover_shifted_data() and recover_original_data() --- .../polynomial-commitments-sampling.md | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index be215ee049..b9e31b3bcc 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -520,6 +520,12 @@ def recover_shifted_data(cell_ids: Sequence[CellID], Sequence[BLSFieldElement], Sequence[BLSFieldElement], BLSFieldElement]: + """ + Given Z(x), return polynomial Q_1(x)=(E*Z)(k*x) and Q_2(x)=Z(k*x) and k^{-1} + """ + shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) + shift_inv = div(BLSFieldElement(1), shift_factor) + extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL @@ -527,15 +533,15 @@ def recover_shifted_data(cell_ids: Sequence[CellID], extended_evaluation_rbo[start:end] = cell extended_evaluation = bit_reversal_permutation(extended_evaluation_rbo) + # Compute (E*Z)(x) extended_evaluation_times_zero = [BLSFieldElement(int(a) * int(b) % BLS_MODULUS) for a, b in zip(zero_poly_eval, extended_evaluation)] extended_evaluations_fft = fft_field(extended_evaluation_times_zero, roots_of_unity_extended, inv=True) - shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) - shift_inv = div(BLSFieldElement(1), shift_factor) - + # Compute (E*Z)(k*x) shifted_extended_evaluation = shift_polynomialcoeff(extended_evaluations_fft, shift_factor) + # Compute Z(k*x) shifted_zero_poly = shift_polynomialcoeff(zero_poly_coeff, shift_factor) eval_shifted_extended_evaluation = fft_field(shifted_extended_evaluation, roots_of_unity_extended) @@ -551,6 +557,10 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle eval_shifted_zero_poly: Sequence[BLSFieldElement], shift_inv: BLSFieldElement, roots_of_unity_extended: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]: + """ + Given Q_1, Q_2 and k^{-1}, compute P(x) + """ + # Compute Q_3 = Q_1(x)/Q_2(x) = P(k*x) eval_shifted_reconstructed_poly = [ div(a, b) for a, b in zip(eval_shifted_extended_evaluation, eval_shifted_zero_poly) @@ -558,6 +568,7 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle shifted_reconstructed_poly = fft_field(eval_shifted_reconstructed_poly, roots_of_unity_extended, inv=True) + # Unshift P(k*x) by k^{-1} to get P(x) reconstructed_poly = shift_polynomialcoeff(shifted_reconstructed_poly, shift_inv) reconstructed_data = bit_reversal_permutation(fft_field(reconstructed_poly, roots_of_unity_extended)) @@ -571,10 +582,11 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle def recover_polynomial(cell_ids: Sequence[CellID], cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]]) -> Polynomial: """ - Recovers a polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. + Recover original polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. This + algorithm uses FFTs to recover cells faster than using Lagrange implementation, as can be seen here: + https://ethresear.ch/t/reed-solomon-erasure-code-recovery-in-n-log-2-n-time-with-ffts/3039 - This algorithm uses FFTs to recover cells faster than using Lagrange implementation. However, - a faster version thanks to Qi Zhou can be found here: + A faster version thanks to Qi Zhou can be found here: https://github.com/ethereum/research/blob/51b530a53bd4147d123ab3e390a9d08605c2cdb8/polynomial_reconstruction/polynomial_reconstruction_danksharding.py Public method. From 212c1fe624f8f84c77f30201ace8045f25ed0227 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 31 Jan 2024 13:32:23 +0200 Subject: [PATCH 11/26] Small improvements on construct_vanishing_polynomial() --- .../polynomial-commitments-sampling.md | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index b9e31b3bcc..3b89f07996 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -477,34 +477,40 @@ def verify_cell_proof_batch(row_commitments_bytes: Sequence[Bytes48], ### `construct_vanishing_polynomial` ```python -def construct_vanishing_polynomial(cell_ids: Sequence[CellID], - cells: Sequence[Cell]) -> Tuple[ - Sequence[BLSFieldElement], - Sequence[BLSFieldElement]]: - missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] +def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ + Sequence[BLSFieldElement], + Sequence[BLSFieldElement]]: + """ + Given the cells that are missing from the data, compute the polynomial that vanishes at every point that + corresponds to a missing field element + """ + # Get the small domain roots_of_unity_reduced = compute_roots_of_unity(CELLS_PER_BLOB) + + # Compute polynomial that vanishes at all the missing cells (over the small domain) short_zero_poly = vanishing_polynomialcoeff([ - roots_of_unity_reduced[reverse_bits(cell_id, CELLS_PER_BLOB)] - for cell_id in missing_cell_ids + roots_of_unity_reduced[reverse_bits(missing_cell_id, CELLS_PER_BLOB)] + for missing_cell_id in missing_cell_ids ]) - zero_poly_coeff = [] - for i in short_zero_poly: - zero_poly_coeff.append(i) - zero_poly_coeff.extend([0] * (FIELD_ELEMENTS_PER_CELL - 1)) - zero_poly_coeff = zero_poly_coeff + [0] * (2 * FIELD_ELEMENTS_PER_BLOB - len(zero_poly_coeff)) + # Extend vanishing polynomial to full domain using the closed form of the vanishing polynomial over a coset + zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) + for (i, coeff) in enumerate(short_zero_poly): + zero_poly_coeff[i * FIELD_ELEMENTS_PER_CELL] = coeff + # Compute evaluations of the extended vanishing polynomial zero_poly_eval = fft_field(zero_poly_coeff, compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) zero_poly_eval_brp = bit_reversal_permutation(zero_poly_eval) - for cell_id in missing_cell_ids: - start = cell_id * FIELD_ELEMENTS_PER_CELL - end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL - assert zero_poly_eval_brp[start:end] == [0] * FIELD_ELEMENTS_PER_CELL - for cell_id in cell_ids: + + # Sanity check + for cell_id in range(CELLS_PER_BLOB): start = cell_id * FIELD_ELEMENTS_PER_CELL end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL - assert all(a != 0 for a in zero_poly_eval_brp[start:end]) + if cell_id in missing_cell_ids: + assert zero_poly_eval_brp[start:end] == [0] * FIELD_ELEMENTS_PER_CELL + else: # cell_id in cell_ids + assert all(a != 0 for a in zero_poly_eval_brp[start:end]) return zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp ``` @@ -593,12 +599,15 @@ def recover_polynomial(cell_ids: Sequence[CellID], """ assert len(cell_ids) == len(cells_bytes) + # Get the extended domain + roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) + + # Convert from bytes to cells cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] assert len(cells) >= CELLS_PER_BLOB // 2 - roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) - - zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(cell_ids, cells) + missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] + zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(missing_cell_ids) eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = \ recover_shifted_data(cell_ids, cells, zero_poly_eval, zero_poly_coeff, roots_of_unity_extended) From d60580bb5267a2d58d819f612bd246c6289db1c0 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Feb 2024 18:09:37 +0800 Subject: [PATCH 12/26] Apply suggestions from code review Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- .../eip7594/polynomial-commitments-sampling.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 3b89f07996..34e7f79533 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -482,7 +482,7 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ Sequence[BLSFieldElement]]: """ Given the cells that are missing from the data, compute the polynomial that vanishes at every point that - corresponds to a missing field element + corresponds to a missing field element. """ # Get the small domain roots_of_unity_reduced = compute_roots_of_unity(CELLS_PER_BLOB) @@ -495,7 +495,7 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ # Extend vanishing polynomial to full domain using the closed form of the vanishing polynomial over a coset zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) - for (i, coeff) in enumerate(short_zero_poly): + for i, coeff in enumerate(short_zero_poly): zero_poly_coeff[i * FIELD_ELEMENTS_PER_CELL] = coeff # Compute evaluations of the extended vanishing polynomial @@ -508,7 +508,7 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ start = cell_id * FIELD_ELEMENTS_PER_CELL end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL if cell_id in missing_cell_ids: - assert zero_poly_eval_brp[start:end] == [0] * FIELD_ELEMENTS_PER_CELL + assert all(a == 0 for a in zero_poly_eval_brp[start:end]) else: # cell_id in cell_ids assert all(a != 0 for a in zero_poly_eval_brp[start:end]) @@ -527,7 +527,7 @@ def recover_shifted_data(cell_ids: Sequence[CellID], Sequence[BLSFieldElement], BLSFieldElement]: """ - Given Z(x), return polynomial Q_1(x)=(E*Z)(k*x) and Q_2(x)=Z(k*x) and k^{-1} + Given Z(x), return polynomial Q_1(x)=(E*Z)(k*x) and Q_2(x)=Z(k*x) and k^{-1}. """ shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) shift_inv = div(BLSFieldElement(1), shift_factor) @@ -564,7 +564,7 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle shift_inv: BLSFieldElement, roots_of_unity_extended: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]: """ - Given Q_1, Q_2 and k^{-1}, compute P(x) + Given Q_1, Q_2 and k^{-1}, compute P(x). """ # Compute Q_3 = Q_1(x)/Q_2(x) = P(k*x) eval_shifted_reconstructed_poly = [ From e3b83d5450d472e12ed7b902b07e9dc613a074e7 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 5 Feb 2024 16:53:28 +0200 Subject: [PATCH 13/26] Add defensive asserts in recover_polynomial() --- specs/_features/eip7594/polynomial-commitments-sampling.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 34e7f79533..efef2779b4 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -598,13 +598,16 @@ def recover_polynomial(cell_ids: Sequence[CellID], Public method. """ assert len(cell_ids) == len(cells_bytes) + # Check we have enough cells to be able to perform the reconstruction + assert CELLS_PER_BLOB / 2 <= len(cell_ids) <= CELLS_PER_BLOB + # Check for duplicates + assert len(cell_ids) == len(set(cell_ids)) # Get the extended domain roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) # Convert from bytes to cells cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] - assert len(cells) >= CELLS_PER_BLOB // 2 missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(missing_cell_ids) From 1e41a6bc698d93131e96cc069931fe630902ccf5 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 5 Feb 2024 16:53:59 +0200 Subject: [PATCH 14/26] New FIELD_ELEMENTS_PER_EXT_BLOB: number of cells in an extended blob --- .../eip7594/polynomial-commitments-sampling.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index efef2779b4..7426025444 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -79,9 +79,10 @@ Cells are the smallest unit of blob data that can come with their own KZG proofs | Name | Value | Description | | - | - | - | +| `FIELD_ELEMENTS_PER_EXT_BLOB` | `2 * FIELD_ELEMENTS_PER_BLOB` | Number of field elements in a Reed-Solomon extended blob | | `FIELD_ELEMENTS_PER_CELL` | `uint64(64)` | Number of field elements in a cell | | `BYTES_PER_CELL` | `FIELD_ELEMENTS_PER_CELL * BYTES_PER_FIELD_ELEMENT` | The number of bytes in a cell | -| `CELLS_PER_BLOB` | `((2 * FIELD_ELEMENTS_PER_BLOB) // FIELD_ELEMENTS_PER_CELL)` | The number of cells in a blob | +| `CELLS_PER_BLOB` | `FIELD_ELEMENTS_PER_EXT_BLOB // FIELD_ELEMENTS_PER_CELL` | The number of cells in a blob | | `RANDOM_CHALLENGE_KZG_CELL_BATCH_DOMAIN` | `b'RCKZGCBATCH__V1_'` | ## Helper functions @@ -355,7 +356,7 @@ def coset_for_cell(cell_id: CellID) -> Cell: """ assert cell_id < CELLS_PER_BLOB roots_of_unity_brp = bit_reversal_permutation( - compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) + compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB) ) return Cell(roots_of_unity_brp[FIELD_ELEMENTS_PER_CELL * cell_id:FIELD_ELEMENTS_PER_CELL * (cell_id + 1)]) ``` @@ -405,7 +406,7 @@ def compute_cells(blob: Blob) -> Vector[Cell, CELLS_PER_BLOB]: polynomial_coeff = polynomial_eval_to_coeff(polynomial) extended_data = fft_field(polynomial_coeff + [0] * FIELD_ELEMENTS_PER_BLOB, - compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) + compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB)) extended_data_rbo = bit_reversal_permutation(extended_data) return [extended_data_rbo[i * FIELD_ELEMENTS_PER_CELL:(i + 1) * FIELD_ELEMENTS_PER_CELL] for i in range(CELLS_PER_BLOB)] @@ -494,13 +495,13 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ ]) # Extend vanishing polynomial to full domain using the closed form of the vanishing polynomial over a coset - zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) + zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_EXT_BLOB) for i, coeff in enumerate(short_zero_poly): zero_poly_coeff[i * FIELD_ELEMENTS_PER_CELL] = coeff # Compute evaluations of the extended vanishing polynomial zero_poly_eval = fft_field(zero_poly_coeff, - compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) + compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB)) zero_poly_eval_brp = bit_reversal_permutation(zero_poly_eval) # Sanity check @@ -532,7 +533,7 @@ def recover_shifted_data(cell_ids: Sequence[CellID], shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) shift_inv = div(BLSFieldElement(1), shift_factor) - extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) + extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_EXT_BLOB) for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL @@ -588,7 +589,7 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle def recover_polynomial(cell_ids: Sequence[CellID], cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]]) -> Polynomial: """ - Recover original polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. This + Recover original polynomial from FIELD_ELEMENTS_PER_EXT_BLOB evaluations, half of which can be missing. This algorithm uses FFTs to recover cells faster than using Lagrange implementation, as can be seen here: https://ethresear.ch/t/reed-solomon-erasure-code-recovery-in-n-log-2-n-time-with-ffts/3039 @@ -604,7 +605,7 @@ def recover_polynomial(cell_ids: Sequence[CellID], assert len(cell_ids) == len(set(cell_ids)) # Get the extended domain - roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) + roots_of_unity_extended = compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB) # Convert from bytes to cells cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] From 4b1106b39c1ffe9604ab2b961a40409e93bad2ef Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 5 Feb 2024 16:59:08 +0200 Subject: [PATCH 15/26] Update specs/_features/eip7594/polynomial-commitments-sampling.md Co-authored-by: Hsiao-Wei Wang --- .../_features/eip7594/polynomial-commitments-sampling.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 7426025444..fb723fb026 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -616,9 +616,12 @@ def recover_polynomial(cell_ids: Sequence[CellID], eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = \ recover_shifted_data(cell_ids, cells, zero_poly_eval, zero_poly_coeff, roots_of_unity_extended) - reconstructed_data = \ - recover_original_data(eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv, - roots_of_unity_extended) + reconstructed_data = recover_original_data( + eval_shifted_extended_evaluation, + eval_shifted_zero_poly, + shift_inv, + roots_of_unity_extended, + ) for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL From f1dd735784c9992fac4260b2c595d9adc8cb2ffc Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 6 Feb 2024 13:10:24 +0200 Subject: [PATCH 16/26] Update specs/_features/eip7594/polynomial-commitments-sampling.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/_features/eip7594/polynomial-commitments-sampling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index fb723fb026..7abc6cb343 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -495,7 +495,7 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ ]) # Extend vanishing polynomial to full domain using the closed form of the vanishing polynomial over a coset - zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_EXT_BLOB) + zero_poly_coeff = [0] * FIELD_ELEMENTS_PER_EXT_BLOB for i, coeff in enumerate(short_zero_poly): zero_poly_coeff[i * FIELD_ELEMENTS_PER_CELL] = coeff From 385b0f35ac3e924e6d1a4fdd46d0e93c4eb1b305 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 6 Feb 2024 13:10:31 +0200 Subject: [PATCH 17/26] Update specs/_features/eip7594/polynomial-commitments-sampling.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/_features/eip7594/polynomial-commitments-sampling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 7abc6cb343..2b96716ade 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -533,7 +533,7 @@ def recover_shifted_data(cell_ids: Sequence[CellID], shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) shift_inv = div(BLSFieldElement(1), shift_factor) - extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_EXT_BLOB) + extended_evaluation_rbo = [0] * FIELD_ELEMENTS_PER_EXT_BLOB for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL From fe344b79d420a3d1020ebc9f254b77b7ed931591 Mon Sep 17 00:00:00 2001 From: Guruprasad Kamath Date: Tue, 6 Feb 2024 17:53:07 +0100 Subject: [PATCH 18/26] correct the location of py.typed --- py.typed => tests/core/pyspec/eth2spec/py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename py.typed => tests/core/pyspec/eth2spec/py.typed (100%) diff --git a/py.typed b/tests/core/pyspec/eth2spec/py.typed similarity index 100% rename from py.typed rename to tests/core/pyspec/eth2spec/py.typed From 4e2578dfe4097e2a6b6284ff6991089057d48776 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 8 Feb 2024 15:32:28 +0100 Subject: [PATCH 19/26] Schedule Deneb for Mainnet As discussed in https://github.com/ethereum/pm/issues/951, schedule Deneb for epoch 269568. --- configs/mainnet.yaml | 2 +- specs/deneb/fork.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index d31634958a..317daa1a45 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -49,7 +49,7 @@ CAPELLA_FORK_VERSION: 0x03000000 CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC # Deneb DENEB_FORK_VERSION: 0x04000000 -DENEB_FORK_EPOCH: 18446744073709551615 +DENEB_FORK_EPOCH: 269568 # March 13, 2024, 01:55:35pm UTC # EIP6110 EIP6110_FORK_VERSION: 0x05000000 # temporary stub EIP6110_FORK_EPOCH: 18446744073709551615 diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 9c314052b1..17af2b15c5 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -29,7 +29,7 @@ Warning: this configuration is not definitive. | Name | Value | | - | - | | `DENEB_FORK_VERSION` | `Version('0x04000000')` | -| `DENEB_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | +| `DENEB_FORK_EPOCH` | `Epoch(269568)` (March 13, 2024, 01:55:35pm UTC) | ## Helper functions From 4878f1da30393582d67e2736a2c058430d05784f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 8 Feb 2024 22:34:59 +0800 Subject: [PATCH 20/26] Set official mainnet `DENEB_FORK_EPOCH` --- README.md | 2 +- configs/mainnet.yaml | 2 +- specs/deneb/fork.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aeda0712d9..1e4ceebf54 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,11 @@ Features are researched and developed in parallel, and then consolidated into se | 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md), [light client](specs/altair/light-client/light-client.md), [networking](specs/altair/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | 2 | **Bellatrix**
(["The Merge"](https://ethereum.org/en/upgrades/merge/)) | `144896` |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| | 3 | **Capella** | `194048` |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/capella/light-client/sync-protocol.md) ([fork](specs/capella/light-client/fork.md), [full node](specs/capella/light-client/full-node.md), [networking](specs/capella/light-client/p2p-interface.md))
    • [Validator additions](specs/capella/validator.md)
    • [P2P networking](specs/capella/p2p-interface.md)
| +| 4 | **Deneb** | `269568` |
  • Core
    • [Beacon Chain changes](specs/deneb/beacon-chain.md)
    • [Deneb fork](specs/deneb/fork.md)
    • [Polynomial commitments](specs/deneb/polynomial-commitments.md)
    • [Fork choice changes](specs/deneb/fork-choice.md)
  • Additions
    • [Light client sync protocol changes](specs/deneb/light-client/sync-protocol.md) ([fork](specs/deneb/light-client/fork.md), [full node](specs/deneb/light-client/full-node.md), [networking](specs/deneb/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/deneb/validator.md)
    • [P2P networking](specs/deneb/p2p-interface.md)
| ### In-development Specifications | Code Name or Topic | Specs | Notes | | - | - | - | -| Deneb (tentative) |
  • Core
    • [Beacon Chain changes](specs/deneb/beacon-chain.md)
    • [Deneb fork](specs/deneb/fork.md)
    • [Polynomial commitments](specs/deneb/polynomial-commitments.md)
    • [Fork choice changes](specs/deneb/fork-choice.md)
  • Additions
    • [Light client sync protocol changes](specs/deneb/light-client/sync-protocol.md) ([fork](specs/deneb/light-client/fork.md), [full node](specs/deneb/light-client/full-node.md), [networking](specs/deneb/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/deneb/validator.md)
    • [P2P networking](specs/deneb/p2p-interface.md)
| | Sharding (outdated) |
  • Core
    • [Beacon Chain changes](specs/_features/sharding/beacon-chain.md)
  • Additions
    • [P2P networking](specs/_features/sharding/p2p-interface.md)
| | Custody Game (outdated) |
  • Core
    • [Beacon Chain changes](specs/_features/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/_features/custody_game/validator.md)
| Dependent on sharding | | Data Availability Sampling (outdated) |
  • Core
    • [Core types and functions](specs/_features/das/das-core.md)
    • [Fork choice changes](specs/_features/das/fork-choice.md)
  • Additions
    • [P2P Networking](specs/_features/das/p2p-interface.md)
    • [Sampling process](specs/_features/das/sampling.md)
|
  • Dependent on sharding
  • [Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)
| diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index d31634958a..cd6972c8be 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -49,7 +49,7 @@ CAPELLA_FORK_VERSION: 0x03000000 CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC # Deneb DENEB_FORK_VERSION: 0x04000000 -DENEB_FORK_EPOCH: 18446744073709551615 +DENEB_FORK_EPOCH: 269568 # March 13, 2024, 01:55:35pm UTC # EIP6110 EIP6110_FORK_VERSION: 0x05000000 # temporary stub EIP6110_FORK_EPOCH: 18446744073709551615 diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 9c314052b1..17af2b15c5 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -29,7 +29,7 @@ Warning: this configuration is not definitive. | Name | Value | | - | - | | `DENEB_FORK_VERSION` | `Version('0x04000000')` | -| `DENEB_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | +| `DENEB_FORK_EPOCH` | `Epoch(269568)` (March 13, 2024, 01:55:35pm UTC) | ## Helper functions From 07bba7a1c55dfaeb26bc1b7dbb03b88d43893706 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 13 Feb 2024 22:23:19 +0800 Subject: [PATCH 21/26] Bump version to `1.4.0-beta.7` --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 2a2e99c7e0..6c30ee1dfb 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.4.0-beta.6 +1.4.0-beta.7 From 7076fb2b759fe2d8135a9c1523af5f90494c59b2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 14 Feb 2024 17:18:05 +0800 Subject: [PATCH 22/26] Handle `integer_squareroot` bound case --- specs/phase0/beacon-chain.md | 3 +++ .../test/phase0/unittests/math/__init__.py | 0 .../unittests/math/test_integer_squareroot.py | 23 +++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/phase0/unittests/math/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/unittests/math/test_integer_squareroot.py diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index bdfb078380..536bba3364 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -178,6 +178,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | +| `MAX_UINT_64` | `uint64(2**64 - 1)` | | `GENESIS_SLOT` | `Slot(0)` | | `GENESIS_EPOCH` | `Epoch(0)` | | `FAR_FUTURE_EPOCH` | `Epoch(2**64 - 1)` | @@ -599,6 +600,8 @@ def integer_squareroot(n: uint64) -> uint64: """ Return the largest integer ``x`` such that ``x**2 <= n``. """ + if n == MAX_UINT_64: + return uint64(4294967295) x = n y = (x + 1) // 2 while y < x: diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/math/__init__.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/math/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/math/test_integer_squareroot.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/math/test_integer_squareroot.py new file mode 100644 index 0000000000..988de1657c --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/math/test_integer_squareroot.py @@ -0,0 +1,23 @@ +import random +from math import isqrt +from eth2spec.test.context import ( + spec_test, + single_phase, + with_all_phases, +) + + +@with_all_phases +@spec_test +@single_phase +def test_integer_squareroot(spec): + values = [0, 100, 2**64 - 2, 2**64 - 1] + for n in values: + uint64_n = spec.uint64(n) + assert spec.integer_squareroot(uint64_n) == isqrt(n) + + rng = random.Random(5566) + for _ in range(10): + n = rng.randint(0, 2**64 - 1) + uint64_n = spec.uint64(n) + assert spec.integer_squareroot(uint64_n) == isqrt(n) From 90afb238eddf37852813784544a7ddc923c637e6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 14 Feb 2024 20:51:50 +0800 Subject: [PATCH 23/26] Update specs/_features/eip7594/polynomial-commitments-sampling.md --- .../_features/eip7594/polynomial-commitments-sampling.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 2b96716ade..9cd862589b 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -613,8 +613,13 @@ def recover_polynomial(cell_ids: Sequence[CellID], missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(missing_cell_ids) - eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = \ - recover_shifted_data(cell_ids, cells, zero_poly_eval, zero_poly_coeff, roots_of_unity_extended) + eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = recover_shifted_data( + cell_ids, + cells, + zero_poly_eval, + zero_poly_coeff, + roots_of_unity_extended, + ) reconstructed_data = recover_original_data( eval_shifted_extended_evaluation, From 42bc6004e58f71960276a86feb10164735652035 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 14 Feb 2024 22:26:58 +0800 Subject: [PATCH 24/26] Add overflow case --- .../test/phase0/unittests/math/test_integer_squareroot.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/math/test_integer_squareroot.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/math/test_integer_squareroot.py index 988de1657c..159f3c9b94 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/math/test_integer_squareroot.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/math/test_integer_squareroot.py @@ -21,3 +21,9 @@ def test_integer_squareroot(spec): n = rng.randint(0, 2**64 - 1) uint64_n = spec.uint64(n) assert spec.integer_squareroot(uint64_n) == isqrt(n) + + try: + spec.integer_squareroot(spec.uint64(2**64)) + assert False + except ValueError: + pass From f020649519eae4d3f2fa70a15c294cdf4cbfdd7d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 15 Feb 2024 00:19:27 +0800 Subject: [PATCH 25/26] add higher_churn_limit_to_lower tests --- .../test/deneb/transition/test_transition.py | 112 ++++++++++++++++++ tests/generators/transition/main.py | 2 + 2 files changed, 114 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/deneb/transition/test_transition.py diff --git a/tests/core/pyspec/eth2spec/test/deneb/transition/test_transition.py b/tests/core/pyspec/eth2spec/test/deneb/transition/test_transition.py new file mode 100644 index 0000000000..04a419c131 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/transition/test_transition.py @@ -0,0 +1,112 @@ +from eth2spec.test.context import ( + ForkMeta, + with_fork_metas, + with_presets, +) +from eth2spec.test.helpers.constants import ( + AFTER_DENEB_PRE_POST_FORKS, + MINIMAL, +) +from eth2spec.test.helpers.keys import pubkeys +from eth2spec.test.helpers.fork_transition import ( + do_fork, + transition_to_next_epoch_and_append_blocks, + transition_until_fork, +) + + +def mock_activated_validators(spec, state, mock_activations): + validator_count = len(state.validators) + for i in range(mock_activations): + index = validator_count + i + validator = spec.Validator( + pubkey=pubkeys[index], + withdrawal_credentials=spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + b'\x00' * 11 + b'\x56' * 20, + activation_eligibility_epoch=0, + activation_epoch=spec.FAR_FUTURE_EPOCH, + exit_epoch=spec.FAR_FUTURE_EPOCH, + withdrawable_epoch=spec.FAR_FUTURE_EPOCH, + effective_balance=spec.MAX_EFFECTIVE_BALANCE, + ) + state.validators.append(validator) + state.balances.append(spec.MAX_EFFECTIVE_BALANCE) + state.previous_epoch_participation.append(spec.ParticipationFlags(0b0000_0000)) + state.current_epoch_participation.append(spec.ParticipationFlags(0b0000_0000)) + state.inactivity_scores.append(0) + state.validators[index].activation_epoch = spec.get_current_epoch(state) + + +@with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=2) + for pre, post in AFTER_DENEB_PRE_POST_FORKS]) +@with_presets([MINIMAL], reason="churn limit update needs enough validators") +def test_higher_churn_limit_to_lower(state, fork_epoch, spec, post_spec, pre_tag, post_tag): + """ + Test if churn limit goes from high to low due to EIP-7514. + """ + # Create high churn limit + mock_activations = post_spec.config.MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT * spec.config.CHURN_LIMIT_QUOTIENT + mock_activated_validators(spec, state, mock_activations) + + transition_until_fork(spec, state, fork_epoch) + + churn_limit_0 = spec.get_validator_churn_limit(state) + assert churn_limit_0 > post_spec.config.MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT + + # check pre state + assert spec.get_current_epoch(state) < fork_epoch + + yield "pre", state + + # irregular state transition to handle fork + blocks = [] + state, block = do_fork(state, spec, post_spec, fork_epoch) + blocks.append(post_tag(block)) + + # check post state + assert spec.get_current_epoch(state) == fork_epoch + + # continue regular state transition with new spec into next epoch + transition_to_next_epoch_and_append_blocks(post_spec, state, post_tag, blocks, only_last_block=True) + + yield "blocks", blocks + yield "post", state + + churn_limit_1 = post_spec.get_validator_activation_churn_limit(state) + assert churn_limit_1 == post_spec.config.MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT + assert churn_limit_1 < churn_limit_0 + + +@with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=2) + for pre, post in AFTER_DENEB_PRE_POST_FORKS]) +@with_presets([MINIMAL], reason="churn limit update needs enough validators") +def test_higher_churn_limit_to_lower__without_block(state, fork_epoch, spec, post_spec, pre_tag, post_tag): + """ + Test if churn limit goes from high to low due to EIP-7514. + """ + # Create high churn limit + mock_activations = post_spec.config.MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT * spec.config.CHURN_LIMIT_QUOTIENT + mock_activated_validators(spec, state, mock_activations) + + transition_until_fork(spec, state, fork_epoch) + + churn_limit_0 = spec.get_validator_churn_limit(state) + assert churn_limit_0 > post_spec.config.MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT + + # check pre state + assert spec.get_current_epoch(state) < fork_epoch + + yield "pre", state + + # irregular state transition to handle fork + # set with_block=False here + state, _ = do_fork(state, spec, post_spec, fork_epoch, with_block=False) + + # check post state + assert spec.get_current_epoch(state) == fork_epoch + + yield "blocks", [] + yield "post", state + + churn_limit_1 = post_spec.get_validator_activation_churn_limit(state) + assert churn_limit_1 == post_spec.config.MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT + assert churn_limit_1 < churn_limit_0 diff --git a/tests/generators/transition/main.py b/tests/generators/transition/main.py index 303f309c26..75ce72b28c 100644 --- a/tests/generators/transition/main.py +++ b/tests/generators/transition/main.py @@ -18,6 +18,7 @@ ) from eth2spec.test.deneb.transition import ( test_operations as test_deneb_operations, + test_transition as test_deneb_transition, ) @@ -47,6 +48,7 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: test_altair_slashing, test_altair_operations, test_deneb_operations, + test_deneb_transition, ) for transition_test_module in all_tests: for pre_fork, post_fork in ALL_PRE_POST_FORKS: From e3d91d8e2a4f319dc4dcc3a5ea7f8264d2b569e0 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 15 Feb 2024 15:00:56 +0800 Subject: [PATCH 26/26] Rename `MAX_UINT_64` -> `UINT64_MAX` --- specs/phase0/beacon-chain.md | 4 ++-- .../pyspec/eth2spec/test/helpers/constants.py | 2 +- .../unittests/test_config_invariants.py | 22 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 536bba3364..572f46246c 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -178,7 +178,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | -| `MAX_UINT_64` | `uint64(2**64 - 1)` | +| `UINT64_MAX` | `uint64(2**64 - 1)` | | `GENESIS_SLOT` | `Slot(0)` | | `GENESIS_EPOCH` | `Epoch(0)` | | `FAR_FUTURE_EPOCH` | `Epoch(2**64 - 1)` | @@ -600,7 +600,7 @@ def integer_squareroot(n: uint64) -> uint64: """ Return the largest integer ``x`` such that ``x**2 <= n``. """ - if n == MAX_UINT_64: + if n == UINT64_MAX: return uint64(4294967295) x = n y = (x + 1) // 2 diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index fc939ce097..a51201906c 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -89,4 +89,4 @@ # # Number # -MAX_UINT_64 = 2**64 - 1 +UINT64_MAX = 2**64 - 1 diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py index 6216ea2f00..c58f817f7d 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py @@ -2,7 +2,7 @@ spec_state_test, with_all_phases, ) -from eth2spec.test.helpers.constants import MAX_UINT_64 +from eth2spec.test.helpers.constants import UINT64_MAX from eth2spec.test.helpers.forks import ( is_post_altair, is_post_bellatrix, ) @@ -16,9 +16,9 @@ def check_bound(value, lower_bound, upper_bound): @with_all_phases @spec_state_test def test_validators(spec, state): - check_bound(spec.VALIDATOR_REGISTRY_LIMIT, 1, MAX_UINT_64) - check_bound(spec.MAX_COMMITTEES_PER_SLOT, 1, MAX_UINT_64) - check_bound(spec.TARGET_COMMITTEE_SIZE, 1, MAX_UINT_64) + check_bound(spec.VALIDATOR_REGISTRY_LIMIT, 1, UINT64_MAX) + check_bound(spec.MAX_COMMITTEES_PER_SLOT, 1, UINT64_MAX) + check_bound(spec.TARGET_COMMITTEE_SIZE, 1, UINT64_MAX) # Note: can be less if you assume stricters bounds on validator set based on total ETH supply maximum_validators_per_committee = ( @@ -30,24 +30,24 @@ def test_validators(spec, state): check_bound(spec.config.MIN_PER_EPOCH_CHURN_LIMIT, 1, spec.VALIDATOR_REGISTRY_LIMIT) check_bound(spec.config.CHURN_LIMIT_QUOTIENT, 1, spec.VALIDATOR_REGISTRY_LIMIT) - check_bound(spec.config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT, spec.TARGET_COMMITTEE_SIZE, MAX_UINT_64) + check_bound(spec.config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT, spec.TARGET_COMMITTEE_SIZE, UINT64_MAX) @with_all_phases @spec_state_test def test_balances(spec, state): assert spec.MAX_EFFECTIVE_BALANCE % spec.EFFECTIVE_BALANCE_INCREMENT == 0 - check_bound(spec.MIN_DEPOSIT_AMOUNT, 1, MAX_UINT_64) - check_bound(spec.MAX_EFFECTIVE_BALANCE, spec.MIN_DEPOSIT_AMOUNT, MAX_UINT_64) - check_bound(spec.MAX_EFFECTIVE_BALANCE, spec.EFFECTIVE_BALANCE_INCREMENT, MAX_UINT_64) + check_bound(spec.MIN_DEPOSIT_AMOUNT, 1, UINT64_MAX) + check_bound(spec.MAX_EFFECTIVE_BALANCE, spec.MIN_DEPOSIT_AMOUNT, UINT64_MAX) + check_bound(spec.MAX_EFFECTIVE_BALANCE, spec.EFFECTIVE_BALANCE_INCREMENT, UINT64_MAX) @with_all_phases @spec_state_test def test_hysteresis_quotient(spec, state): - check_bound(spec.HYSTERESIS_QUOTIENT, 1, MAX_UINT_64) + check_bound(spec.HYSTERESIS_QUOTIENT, 1, UINT64_MAX) check_bound(spec.HYSTERESIS_DOWNWARD_MULTIPLIER, 1, spec.HYSTERESIS_QUOTIENT) - check_bound(spec.HYSTERESIS_UPWARD_MULTIPLIER, spec.HYSTERESIS_QUOTIENT, MAX_UINT_64) + check_bound(spec.HYSTERESIS_UPWARD_MULTIPLIER, spec.HYSTERESIS_QUOTIENT, UINT64_MAX) @with_all_phases @@ -68,7 +68,7 @@ def test_time(spec, state): assert spec.SLOTS_PER_EPOCH <= spec.SLOTS_PER_HISTORICAL_ROOT assert spec.MIN_SEED_LOOKAHEAD < spec.MAX_SEED_LOOKAHEAD assert spec.SLOTS_PER_HISTORICAL_ROOT % spec.SLOTS_PER_EPOCH == 0 - check_bound(spec.SLOTS_PER_HISTORICAL_ROOT, spec.SLOTS_PER_EPOCH, MAX_UINT_64) + check_bound(spec.SLOTS_PER_HISTORICAL_ROOT, spec.SLOTS_PER_EPOCH, UINT64_MAX) check_bound(spec.MIN_ATTESTATION_INCLUSION_DELAY, 1, spec.SLOTS_PER_EPOCH)