EIP-7916: ProgressiveList and tests

Cargo.toml

@@ -285,3 +285,9 @@ debug = true
 
 [patch.crates-io]
 quick-protobuf = { git = "https://github.com/sigp/quick-protobuf.git", rev = "681f413312404ab6e51f0b46f39b0075c6f4ebfd" }
+# FIXME(sproul): REMOVE patch
+milhouse = { git = "https://github.com/sigp/milhouse", branch = "progressive-list" }
+ethereum_ssz = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" }
+ethereum_ssz_derive = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" }
+tree_hash = { git = "https://github.com/sigp/tree_hash", branch = "progressive" }
+tree_hash_derive = { git = "https://github.com/sigp/tree_hash", branch = "progressive" }

consensus/types/src/lib.rs

@@ -173,6 +173,7 @@ pub use bls::{
 };
 pub use context_deserialize::{ContextDeserialize, context_deserialize};
 pub use fixed_bytes::FixedBytesExtended;
-pub use milhouse::{self, List, Vector};
+pub use milhouse::{self, List, ProgressiveList, Vector};
+pub use ssz::ProgressiveBitList;
 pub use ssz_types::{BitList, BitVector, FixedVector, VariableList, typenum, typenum::Unsigned};
 pub use superstruct::superstruct;

testing/ef_tests/Cargo.toml

@@ -19,6 +19,7 @@ compare_fields = { workspace = true }
 context_deserialize = { workspace = true }
 educe = { workspace = true }
 eth2_network_config = { workspace = true }
+ethereum_hashing = { workspace = true } # FIXME(sproul): remove
 ethereum_ssz = { workspace = true }
 ethereum_ssz_derive = { workspace = true }
 execution_layer = { workspace = true }

testing/ef_tests/Makefile

@@ -1,6 +1,6 @@
 # To download/extract nightly tests, run:
 # CONSENSUS_SPECS_TEST_VERSION=nightly make
-CONSENSUS_SPECS_TEST_VERSION ?= v1.6.0-beta.1
+CONSENSUS_SPECS_TEST_VERSION ?= v1.6.1
 REPO_NAME := consensus-spec-tests
 OUTPUT_DIR := ./$(REPO_NAME)
 

testing/ef_tests/check_all_files_accessed.py

@@ -51,7 +51,6 @@
     "tests/.*/fulu/ssz_static/MatrixEntry/.*",
     # EIP-7916 is still in draft and hasn't been implemented yet https://eips.ethereum.org/EIPS/eip-7916
     "tests/general/phase0/ssz_generic/progressive_bitlist",
-    "tests/general/phase0/ssz_generic/basic_progressive_list",
     "tests/general/phase0/ssz_generic/containers/.*/ProgressiveBitsStruct.*",
     "tests/general/phase0/ssz_generic/containers/.*/ProgressiveTestStruct.*",
     "tests/general/phase0/ssz_generic/progressive_containers/.*",

testing/ef_tests/src/cases/ssz_generic.rs

@@ -10,7 +10,13 @@ use ssz_derive::{Decode, Encode};
 use tree_hash::TreeHash;
 use tree_hash_derive::TreeHash;
 use types::typenum::*;
-use types::{BitList, BitVector, FixedVector, ForkName, VariableList, Vector};
+use types::{
+    BitList, BitVector, FixedVector, ForkName, List, ProgressiveBitList, ProgressiveList,
+    VariableList, Vector,
+};
+
+type U1280 = op!(U128 * U10);
+type U1281 = op!(U1280 + U1);
 
 #[derive(Debug, Clone, Deserialize)]
 #[context_deserialize(ForkName)]
@@ -111,8 +117,12 @@ macro_rules! type_dispatch {
             "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*),
             "ComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ComplexTestStruct>, $($rest)*),
             "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*),
-            // EIP-7916 is still in draft and hasn't been implemented yet https://eips.ethereum.org/EIPS/eip-7916
-            "ProgressiveTestStruct" | "ProgressiveBitsStruct" => Err(Error::SkippedKnownFailure),
+            "ProgressiveBitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveBitsStruct>, $($rest)*),
+            "ProgressiveSingleFieldContainerTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveSingleFieldContainerTestStruct>, $($rest)*),
+            "ProgressiveSingleListContainerTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveSingleListContainerTestStruct>, $($rest)*),
+            "ProgressiveVarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveVarTestStruct>, $($rest)*),
+            "ProgressiveComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveComplexTestStruct>, $($rest)*),
+            "ProgressiveTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveTestStruct>, $($rest)*),
             _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
         }
     };
@@ -157,6 +167,17 @@ impl Case for SszGeneric {
                     [length => typenum]
                 )?;
             }
+            "basic_progressive_list" => {
+                let elem_ty = parts[1];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    ProgressiveList,
+                    <>,
+                    [elem_ty => primitive_type]
+                )?;
+            }
             "bitlist" => {
                 let mut limit = parts[1];
 
@@ -184,6 +205,14 @@ impl Case for SszGeneric {
                     [length => typenum]
                 )?;
             }
+            "progressive_bitlist" => {
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    ProgressiveBitList,
+                    <>,
+                )?;
+            }
             "boolean" => {
                 ssz_generic_test::<bool>(&self.path, fork_name)?;
             }
@@ -209,6 +238,17 @@ impl Case for SszGeneric {
                     [type_name => test_container]
                 )?;
             }
+            "progressive_containers" => {
+                let type_name = parts[0];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    _,
+                    <>,
+                    [type_name => test_container]
+                )?;
+            }
             _ => panic!("unsupported handler: {}", self.handler_name),
         }
         Ok(())
@@ -300,6 +340,15 @@ struct ComplexTestStruct {
     G: Vector<VarTestStruct, U2>,
 }
 
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[context_deserialize(ForkName)]
+struct ProgressiveTestStruct {
+    A: ProgressiveList<u8>,
+    B: ProgressiveList<u64>,
+    C: ProgressiveList<SmallTestStruct>,
+    D: ProgressiveList<ProgressiveList<VarTestStruct>>,
+}
+
 #[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
 #[context_deserialize(ForkName)]
 struct BitsStruct {
@@ -310,6 +359,69 @@ struct BitsStruct {
     E: BitVector<U8>,
 }
 
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[context_deserialize(ForkName)]
+struct ProgressiveBitsStruct {
+    A: BitVector<U256>,
+    B: BitList<U256>,
+    C: ProgressiveBitList,
+    D: BitVector<U257>,
+    E: BitList<U257>,
+    F: ProgressiveBitList,
+    G: BitVector<U1280>,
+    H: BitList<U1280>,
+    I: ProgressiveBitList,
+    J: BitVector<U1281>,
+    K: BitList<U1281>,
+    L: ProgressiveBitList,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(struct_behaviour = "progressive_container", active_fields(1))]
+#[context_deserialize(ForkName)]
+struct ProgressiveSingleFieldContainerTestStruct {
+    A: u8,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(
+    struct_behaviour = "progressive_container",
+    active_fields(0, 0, 0, 0, 1)
+)]
+#[context_deserialize(ForkName)]
+struct ProgressiveSingleListContainerTestStruct {
+    C: ProgressiveBitList,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(
+    struct_behaviour = "progressive_container",
+    active_fields(1, 0, 1, 0, 1)
+)]
+#[context_deserialize(ForkName)]
+struct ProgressiveVarTestStruct {
+    A: u8,
+    B: List<u16, U123>,
+    C: ProgressiveBitList,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(
+    struct_behaviour = "progressive_container",
+    active_fields(1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1)
+)]
+#[context_deserialize(ForkName)]
+struct ProgressiveComplexTestStruct {
+    A: u8,
+    B: List<u16, U123>,
+    C: ProgressiveBitList,
+    D: ProgressiveList<u64>,
+    E: ProgressiveList<SmallTestStruct>,
+    F: ProgressiveList<ProgressiveList<VarTestStruct>>,
+    G: List<ProgressiveSingleFieldContainerTestStruct, U10>,
+    H: ProgressiveList<ProgressiveVarTestStruct>,
+}
+
 fn byte_list_from_hex_str<'de, D, N: Unsigned>(
     deserializer: D,
 ) -> Result<VariableList<u8, N>, D::Error>

testing/ef_tests/src/handler.rs

@@ -1135,13 +1135,19 @@ impl<H: TypeName> Handler for SszGenericHandler<H> {
 // Supported SSZ generic handlers
 pub struct BasicVector;
 type_name!(BasicVector, "basic_vector");
+pub struct BasicProgressiveList;
+type_name!(BasicProgressiveList, "basic_progressive_list");
 pub struct Bitlist;
 type_name!(Bitlist, "bitlist");
 pub struct Bitvector;
 type_name!(Bitvector, "bitvector");
+pub struct ProgressiveBitlist;
+type_name!(ProgressiveBitlist, "progressive_bitlist");
 pub struct Boolean;
 type_name!(Boolean, "boolean");
 pub struct Uints;
 type_name!(Uints, "uints");
 pub struct Containers;
 type_name!(Containers, "containers");
+pub struct ProgressiveContainers;
+type_name!(ProgressiveContainers, "progressive_containers");

testing/ef_tests/tests/tests.rs

@@ -733,6 +733,13 @@ fn ssz_generic() {
     SszGenericHandler::<Containers>::default().run();
 }
 
+#[test]
+fn ssz_generic_progressive() {
+    SszGenericHandler::<BasicProgressiveList>::default().run();
+    SszGenericHandler::<ProgressiveBitlist>::default().run();
+    SszGenericHandler::<ProgressiveContainers>::default().run();
+}
+
 #[test]
 fn epoch_processing_justification_and_finalization() {
     EpochProcessingHandler::<MinimalEthSpec, JustificationAndFinalization>::default().run();