diff --git a/noir-projects/aztec-nr/aztec/src/ephemeral/mod.nr b/noir-projects/aztec-nr/aztec/src/ephemeral/mod.nr index 2b64fe54f4a3..8a666c3366da 100644 --- a/noir-projects/aztec-nr/aztec/src/ephemeral/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/ephemeral/mod.nr @@ -1,6 +1,7 @@ -use crate::oracle::ephemeral; +use crate::oracle::ephemeral_oracles; use crate::protocol::traits::{Deserialize, Serialize}; use crate::protocol::utils::{reader::Reader, writer::Writer}; +use crate::unconstrained_array::{ArrayOracles, UnconstrainedArray}; /// A dynamically sized array that exists only during a single contract call frame. /// @@ -23,98 +24,43 @@ use crate::protocol::utils::{reader::Reader, writer::Writer}; /// For data that must be shared across all frames of the same contract (private and utility) within one top-level PXE /// call (transaction simulation or utility call) but not persisted, use /// [`TransientArray`](crate::transient::TransientArray). -pub struct EphemeralArray { - pub slot: Field, -} +pub type EphemeralArray = UnconstrainedArray; -impl EphemeralArray { - /// Returns a handle to an ephemeral array at the given slot, which may already contain data (e.g. populated - /// by an oracle). - pub unconstrained fn at(slot: Field) -> Self { - Self { slot } - } +pub struct EphemeralOracles {} - /// Returns an empty ephemeral array at the given slot, clearing any pre-existing data. - pub unconstrained fn empty_at(slot: Field) -> Self { - Self::at(slot).clear() +impl ArrayOracles for EphemeralOracles { + unconstrained fn len_oracle(slot: Field) -> u32 { + ephemeral_oracles::len_oracle(slot) } - /// Returns the number of elements stored in the array. - pub unconstrained fn len(self) -> u32 { - ephemeral::len_oracle(self.slot) + unconstrained fn push_oracle(slot: Field, values: [Field; N]) -> u32 { + ephemeral_oracles::push_oracle(slot, values) } - /// Stores a value at the end of the array. - pub unconstrained fn push(self, value: T) - where - T: Serialize, - { - let serialized = value.serialize(); - let _ = ephemeral::push_oracle(self.slot, serialized); + unconstrained fn pop_oracle(slot: Field) -> [Field; N] { + ephemeral_oracles::pop_oracle(slot) } - /// Removes and returns the last element. Panics if the array is empty. - pub unconstrained fn pop(self) -> T - where - T: Deserialize, - { - let serialized = ephemeral::pop_oracle(self.slot); - Deserialize::deserialize(serialized) + unconstrained fn get_oracle(slot: Field, index: u32) -> [Field; N] { + ephemeral_oracles::get_oracle(slot, index) } - /// Retrieves the value stored at `index`. Panics if the index is out of bounds. - pub unconstrained fn get(self, index: u32) -> T - where - T: Deserialize, - { - let serialized = ephemeral::get_oracle(self.slot, index); - Deserialize::deserialize(serialized) + unconstrained fn set_oracle(slot: Field, index: u32, values: [Field; N]) { + ephemeral_oracles::set_oracle(slot, index, values) } - /// Overwrites the value stored at `index`. Panics if the index is out of bounds. - pub unconstrained fn set(self, index: u32, value: T) - where - T: Serialize, - { - let serialized = value.serialize(); - ephemeral::set_oracle(self.slot, index, serialized); + unconstrained fn remove_oracle(slot: Field, index: u32) { + ephemeral_oracles::remove_oracle(slot, index) } - /// Removes the element at `index`, shifting subsequent elements backward. Panics if out of bounds. - pub unconstrained fn remove(self, index: u32) { - ephemeral::remove_oracle(self.slot, index); - } - - /// Removes all elements from the array and returns self for chaining. - /// - /// Prefer [`EphemeralArray::empty_at`] when the intent is to start with a fresh array. - pub unconstrained fn clear(self) -> Self { - ephemeral::clear_oracle(self.slot); - self - } - - /// Calls a function on each element of the array. - /// - /// The function `f` is called once with each array value and its corresponding index. Iteration proceeds - /// backwards so that it is safe to remove the current element (and only the current element) inside the - /// callback. - /// - /// It is **not** safe to push new elements from inside the callback. - pub unconstrained fn for_each(self, f: unconstrained fn[Env](u32, T) -> ()) - where - T: Deserialize, - { - let mut i = self.len(); - while i > 0 { - i -= 1; - f(i, self.get(i)); - } + unconstrained fn clear_oracle(slot: Field) { + ephemeral_oracles::clear_oracle(slot) } } -/// Serializes an `EphemeralArray` as its slot identifier, allowing oracle function signatures to use -/// `EphemeralArray` instead of opaque `Field` slots. -impl Serialize for EphemeralArray { +/// Serializes an `EphemeralArray` as its slot, allowing oracle function signatures to use ephemeral array types +/// instead of opaque `Field` slots. +impl Serialize for UnconstrainedArray { let N: u32 = 1; fn serialize(self) -> [Field; Self::N] { @@ -127,8 +73,7 @@ impl Serialize for EphemeralArray { } /// Deserializes a single Field into an `EphemeralArray` handle, treating the field value as the slot identifier. -/// This is the inverse of [`Serialize`]. -impl Deserialize for EphemeralArray { +impl Deserialize for UnconstrainedArray { let N: u32 = 1; fn deserialize(fields: [Field; Self::N]) -> Self { @@ -140,279 +85,5 @@ impl Deserialize for EphemeralArray { } } -mod test { - use crate::test::helpers::test_environment::TestEnvironment; - use crate::test::mocks::MockStruct; - use super::EphemeralArray; - - global SLOT: Field = 1230; - global OTHER_SLOT: Field = 5670; - - #[test] - unconstrained fn empty_array() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array: EphemeralArray = EphemeralArray::at(SLOT); - assert_eq(array.len(), 0); - }); - } - - #[test(should_fail_with = "out of bounds")] - unconstrained fn empty_array_read() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - let _: Field = array.get(0); - }); - } - - #[test(should_fail_with = "is empty")] - unconstrained fn empty_array_pop() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - let _: Field = array.pop(); - }); - } - - #[test] - unconstrained fn array_push() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - array.push(5); - - assert_eq(array.len(), 1); - assert_eq(array.get(0), 5); - }); - } - - #[test(should_fail_with = "out of bounds")] - unconstrained fn read_past_len() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - array.push(5); - - let _ = array.get(1); - }); - } - - #[test] - unconstrained fn array_pop() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - array.push(5); - array.push(10); - - let popped: Field = array.pop(); - assert_eq(popped, 10); - assert_eq(array.len(), 1); - assert_eq(array.get(0), 5); - }); - } - - #[test] - unconstrained fn array_set() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - array.push(5); - array.set(0, 99); - assert_eq(array.get(0), 99); - }); - } - - #[test] - unconstrained fn array_remove_last() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - array.push(5); - array.remove(0); - assert_eq(array.len(), 0); - }); - } - - #[test] - unconstrained fn array_remove_some() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - - array.push(7); - array.push(8); - array.push(9); - - assert_eq(array.len(), 3); - - array.remove(1); - - assert_eq(array.len(), 2); - assert_eq(array.get(0), 7); - assert_eq(array.get(1), 9); - }); - } - - #[test] - unconstrained fn array_remove_all() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - - array.push(7); - array.push(8); - array.push(9); - - array.remove(1); - array.remove(1); - array.remove(0); - - assert_eq(array.len(), 0); - }); - } - - #[test] - unconstrained fn for_each_called_with_all_elements() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - - array.push(4); - array.push(5); - array.push(6); - - let called_with = &mut BoundedVec::<(u32, Field), 3>::new(); - array.for_each(|index, value| { called_with.push((index, value)); }); - - assert_eq(called_with.len(), 3); - assert(called_with.any(|(index, value)| (index == 0) & (value == 4))); - assert(called_with.any(|(index, value)| (index == 1) & (value == 5))); - assert(called_with.any(|(index, value)| (index == 2) & (value == 6))); - }); - } - - #[test] - unconstrained fn for_each_remove_some() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - - array.push(4); - array.push(5); - array.push(6); - - array.for_each(|index, _| { - if index == 1 { - array.remove(index); - } - }); - - assert_eq(array.len(), 2); - assert_eq(array.get(0), 4); - assert_eq(array.get(1), 6); - }); - } - - #[test] - unconstrained fn for_each_remove_all() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = EphemeralArray::at(SLOT); - - array.push(4); - array.push(5); - array.push(6); - - array.for_each(|index, _| { array.remove(index); }); - - assert_eq(array.len(), 0); - }); - } - - #[test] - unconstrained fn different_slots_are_isolated() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array_a = EphemeralArray::at(SLOT); - let array_b = EphemeralArray::at(OTHER_SLOT); - - array_a.push(10); - array_a.push(20); - array_b.push(99); - - assert_eq(array_a.len(), 2); - assert_eq(array_a.get(0), 10); - assert_eq(array_a.get(1), 20); - - assert_eq(array_b.len(), 1); - assert_eq(array_b.get(0), 99); - }); - } - - #[test] - unconstrained fn works_with_multi_field_type() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array: EphemeralArray = EphemeralArray::at(SLOT); - - let a = MockStruct::new(5, 6); - let b = MockStruct::new(7, 8); - array.push(a); - array.push(b); - - assert_eq(array.len(), 2); - assert_eq(array.get(0), a); - assert_eq(array.get(1), b); - - let popped: MockStruct = array.pop(); - assert_eq(popped, b); - assert_eq(array.len(), 1); - }); - } - - #[test] - unconstrained fn empty_at_wipes_previous_data() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array: EphemeralArray = EphemeralArray::at(SLOT); - array.push(1); - assert_eq(array.len(), 1); - - let fresh: EphemeralArray = EphemeralArray::empty_at(SLOT); - assert_eq(fresh.len(), 0); - }); - } - - #[test] - unconstrained fn clear_returns_self() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array: EphemeralArray = EphemeralArray::at(SLOT).clear(); - assert_eq(array.len(), 0); - - array.push(42); - assert_eq(array.len(), 1); - assert_eq(array.get(0), 42); - }); - } - - #[test] - unconstrained fn clear_wipes_previous_data() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array: EphemeralArray = EphemeralArray::at(SLOT); - array.push(1); - array.push(2); - array.push(3); - assert_eq(array.len(), 3); - - // Clear the same slot, previous data should be gone. - let fresh: EphemeralArray = EphemeralArray::at(SLOT).clear(); - assert_eq(fresh.len(), 0); - fresh.push(4); - assert_eq(fresh.get(0), 4); - }); - } -} +#[crate::unconstrained_array::test_suite::unconstrained_array_tests(quote { crate::ephemeral::EphemeralOracles })] +mod test {} diff --git a/noir-projects/aztec-nr/aztec/src/lib.nr b/noir-projects/aztec-nr/aztec/src/lib.nr index 51d250108832..387f820d22da 100644 --- a/noir-projects/aztec-nr/aztec/src/lib.nr +++ b/noir-projects/aztec-nr/aztec/src/lib.nr @@ -39,6 +39,7 @@ pub mod nullifier; pub mod oracle; pub mod state_vars; pub mod capsules; +pub mod unconstrained_array; pub mod ephemeral; pub mod transient; pub mod event; diff --git a/noir-projects/aztec-nr/aztec/src/oracle/ephemeral.nr b/noir-projects/aztec-nr/aztec/src/oracle/ephemeral_oracles.nr similarity index 84% rename from noir-projects/aztec-nr/aztec/src/oracle/ephemeral.nr rename to noir-projects/aztec-nr/aztec/src/oracle/ephemeral_oracles.nr index 9f2721fc53c8..cb053b599841 100644 --- a/noir-projects/aztec-nr/aztec/src/oracle/ephemeral.nr +++ b/noir-projects/aztec-nr/aztec/src/oracle/ephemeral_oracles.nr @@ -8,15 +8,15 @@ #[oracle(aztec_utl_pushEphemeral)] pub(crate) unconstrained fn push_oracle(slot: Field, values: [Field; N]) -> u32 {} -/// Removes and returns the last serialized element from the ephemeral array. +/// Removes and returns the last serialized element from the ephemeral array. Panics if the array is empty. #[oracle(aztec_utl_popEphemeral)] pub(crate) unconstrained fn pop_oracle(slot: Field) -> [Field; N] {} -/// Returns the serialized element at the given index. +/// Returns the serialized element at the given index. Panics if `index` is out of bounds. #[oracle(aztec_utl_getEphemeral)] pub(crate) unconstrained fn get_oracle(slot: Field, index: u32) -> [Field; N] {} -/// Overwrites the serialized element at the given index. +/// Overwrites the serialized element at the given index. Panics if `index` is out of bounds. #[oracle(aztec_utl_setEphemeral)] pub(crate) unconstrained fn set_oracle(slot: Field, index: u32, values: [Field; N]) {} @@ -24,7 +24,7 @@ pub(crate) unconstrained fn set_oracle(slot: Field, index: u32, valu #[oracle(aztec_utl_getEphemeralLen)] pub(crate) unconstrained fn len_oracle(slot: Field) -> u32 {} -/// Removes the element at the given index, shifting subsequent elements backward. +/// Removes the element at the given index, shifting subsequent elements backward. Panics if `index` is out of bounds. #[oracle(aztec_utl_removeEphemeral)] pub(crate) unconstrained fn remove_oracle(slot: Field, index: u32) {} diff --git a/noir-projects/aztec-nr/aztec/src/oracle/mod.nr b/noir-projects/aztec-nr/aztec/src/oracle/mod.nr index 85953142230d..6d677346a72f 100644 --- a/noir-projects/aztec-nr/aztec/src/oracle/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/oracle/mod.nr @@ -7,8 +7,8 @@ pub mod block_header; pub mod call_private_function; pub mod call_utility_function; pub mod capsules; -pub mod ephemeral; -pub mod transient; +pub(crate) mod ephemeral_oracles; +pub(crate) mod transient_oracles; pub mod contract_sync; pub mod public_call; pub mod tx_phase; diff --git a/noir-projects/aztec-nr/aztec/src/oracle/transient.nr b/noir-projects/aztec-nr/aztec/src/oracle/transient_oracles.nr similarity index 84% rename from noir-projects/aztec-nr/aztec/src/oracle/transient.nr rename to noir-projects/aztec-nr/aztec/src/oracle/transient_oracles.nr index c7db29c51b2d..9c76dca0e635 100644 --- a/noir-projects/aztec-nr/aztec/src/oracle/transient.nr +++ b/noir-projects/aztec-nr/aztec/src/oracle/transient_oracles.nr @@ -9,15 +9,15 @@ #[oracle(aztec_utl_pushTransient)] pub(crate) unconstrained fn push_oracle(slot: Field, values: [Field; N]) -> u32 {} -/// Removes and returns the last serialized element from the transient array. +/// Removes and returns the last serialized element from the transient array. Panics if the array is empty. #[oracle(aztec_utl_popTransient)] pub(crate) unconstrained fn pop_oracle(slot: Field) -> [Field; N] {} -/// Returns the serialized element at the given index. +/// Returns the serialized element at the given index. Panics if `index` is out of bounds. #[oracle(aztec_utl_getTransient)] pub(crate) unconstrained fn get_oracle(slot: Field, index: u32) -> [Field; N] {} -/// Overwrites the serialized element at the given index. +/// Overwrites the serialized element at the given index. Panics if `index` is out of bounds. #[oracle(aztec_utl_setTransient)] pub(crate) unconstrained fn set_oracle(slot: Field, index: u32, values: [Field; N]) {} @@ -25,7 +25,7 @@ pub(crate) unconstrained fn set_oracle(slot: Field, index: u32, valu #[oracle(aztec_utl_getTransientLen)] pub(crate) unconstrained fn len_oracle(slot: Field) -> u32 {} -/// Removes the element at the given index, shifting subsequent elements backward. +/// Removes the element at the given index, shifting subsequent elements backward. Panics if `index` is out of bounds. #[oracle(aztec_utl_removeTransient)] pub(crate) unconstrained fn remove_oracle(slot: Field, index: u32) {} diff --git a/noir-projects/aztec-nr/aztec/src/transient/mod.nr b/noir-projects/aztec-nr/aztec/src/transient/mod.nr index d143b0c294ed..569f0731e252 100644 --- a/noir-projects/aztec-nr/aztec/src/transient/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/transient/mod.nr @@ -1,6 +1,6 @@ -use crate::oracle::transient; +use crate::oracle::transient_oracles; use crate::protocol::traits::{Deserialize, Serialize}; -use crate::protocol::utils::{reader::Reader, writer::Writer}; +use crate::unconstrained_array::{ArrayOracles, UnconstrainedArray}; /// A dynamically sized array that lives for the duration of a single top-level PXE call. /// @@ -29,424 +29,113 @@ use crate::protocol::utils::{reader::Reader, writer::Writer}; /// Use this to pass not-to-be-persisted data between a contract's own frames (private or utility) within one top-level /// PXE call. For data confined to a single call frame, prefer [`EphemeralArray`](crate::ephemeral::EphemeralArray). /// For data that must persist indefinitely, use [`CapsuleArray`](crate::capsules::CapsuleArray). -pub struct TransientArray { - pub slot: Field, +pub type TransientArray = UnconstrainedArray; + +/// A single value that lives for the duration of a single top-level PXE call. +/// +/// Shared across all call frames of the same contract (private and utility alike) within one top-level PXE call. +/// A contract can only ever access its own transient values; other contracts called within the same top-level PXE call +/// cannot see them. +/// +/// Values share the slot space with [`TransientArray`]s: a value and an array at the same slot alias each other, so +/// use distinct slots. +pub struct TransientValue { + slot: Field, } -impl TransientArray { - /// Returns a handle to a transient array at the given slot, which may already contain data pushed by an earlier - /// frame of the same contract in this top-level PXE call. +impl TransientValue { + /// Returns a handle to the value at the given slot, which may already hold data stored by an earlier frame. pub unconstrained fn at(slot: Field) -> Self { Self { slot } } - /// Returns the number of elements stored in the array. - pub unconstrained fn len(self) -> u32 { - transient::len_oracle(self.slot) - } - - /// Stores a value at the end of the array. - pub unconstrained fn push(self, value: T) + /// Stores a value at the given slot. + pub unconstrained fn store(self, value: T) where T: Serialize, { - let serialized = value.serialize(); - let _ = transient::push_oracle(self.slot, serialized); + transient_oracles::clear_oracle(self.slot); + let _ = transient_oracles::push_oracle(self.slot, value.serialize()); } - /// Removes and returns the last element. Panics if the array is empty. - pub unconstrained fn pop(self) -> T + /// Returns the value previously stored at the slot with [`TransientValue::store`], or `Option::none()` if the + /// slot holds no value. + pub unconstrained fn read(self) -> Option where T: Deserialize, { - let serialized = transient::pop_oracle(self.slot); - Deserialize::deserialize(serialized) + if transient_oracles::len_oracle(self.slot) == 0 { + Option::none() + } else { + let serialized = transient_oracles::get_oracle(self.slot, 0); + Option::some(Deserialize::deserialize(serialized)) + } } - /// Retrieves the value stored at `index`. Panics if the index is out of bounds. - pub unconstrained fn get(self, index: u32) -> T - where - T: Deserialize, - { - let serialized = transient::get_oracle(self.slot, index); - Deserialize::deserialize(serialized) + /// Deletes the stored value. Does nothing if the slot holds no value. + pub unconstrained fn delete(self) { + transient_oracles::clear_oracle(self.slot) } +} - /// Overwrites the value stored at `index`. Panics if the index is out of bounds. - pub unconstrained fn set(self, index: u32, value: T) - where - T: Serialize, - { - let serialized = value.serialize(); - transient::set_oracle(self.slot, index, serialized); - } +pub struct TransientOracles {} - /// Removes the element at `index`, shifting subsequent elements backward. Panics if out of bounds. - pub unconstrained fn remove(self, index: u32) { - transient::remove_oracle(self.slot, index); +impl ArrayOracles for TransientOracles { + unconstrained fn len_oracle(slot: Field) -> u32 { + transient_oracles::len_oracle(slot) } - /// Removes all elements from the array and returns self for chaining (e.g. `TransientArray::at(slot).clear()` - /// to get a guaranteed-empty array at a given slot). - pub unconstrained fn clear(self) -> Self { - transient::clear_oracle(self.slot); - self + unconstrained fn push_oracle(slot: Field, values: [Field; N]) -> u32 { + transient_oracles::push_oracle(slot, values) } - /// Calls a function on each element of the array. - /// - /// Iteration proceeds backwards so it is safe to remove the current element (and only the current element) inside - /// the callback. It is **not** safe to push new elements from inside the callback. - pub unconstrained fn for_each(self, f: unconstrained fn[Env](u32, T) -> ()) - where - T: Deserialize, - { - let mut i = self.len(); - while i > 0 { - i -= 1; - f(i, self.get(i)); - } + unconstrained fn pop_oracle(slot: Field) -> [Field; N] { + transient_oracles::pop_oracle(slot) } -} -/// Serializes a `TransientArray` as its slot identifier, allowing oracle function signatures to use `TransientArray` -/// instead of opaque `Field` slots. -impl Serialize for TransientArray { - let N: u32 = 1; - - fn serialize(self) -> [Field; Self::N] { - [self.slot] + unconstrained fn get_oracle(slot: Field, index: u32) -> [Field; N] { + transient_oracles::get_oracle(slot, index) } - fn stream_serialize(self, writer: &mut Writer) { - writer.write(self.slot); + unconstrained fn set_oracle(slot: Field, index: u32, values: [Field; N]) { + transient_oracles::set_oracle(slot, index, values) } -} -/// Deserializes a single Field into a `TransientArray` handle, treating the field value as the slot identifier. -/// This is the inverse of [`Serialize`]. -impl Deserialize for TransientArray { - let N: u32 = 1; - - fn deserialize(fields: [Field; Self::N]) -> Self { - Self { slot: fields[0] } - } - - fn stream_deserialize(reader: &mut Reader) -> Self { - Self { slot: reader.read() } + unconstrained fn remove_oracle(slot: Field, index: u32) { + transient_oracles::remove_oracle(slot, index) } -} - -/// Stores a single value at `slot`, overwriting any value previously stored there. -pub unconstrained fn store(slot: Field, value: T) -where - T: Serialize, -{ - let array: TransientArray = TransientArray::at(slot); - array.clear().push(value); -} -/// Returns the value previously stored at `slot` with [`store`], or `Option::none()` if the slot holds no value. -pub unconstrained fn load(slot: Field) -> Option -where - T: Deserialize, -{ - let array: TransientArray = TransientArray::at(slot); - if array.len() == 0 { - Option::none() - } else { - Option::some(array.get(0)) + unconstrained fn clear_oracle(slot: Field) { + transient_oracles::clear_oracle(slot) } } -/// Deletes the value stored at `slot`. Does nothing if the slot is already empty. -pub unconstrained fn delete(slot: Field) { - transient::clear_oracle(slot); -} +#[crate::unconstrained_array::test_suite::unconstrained_array_tests(quote { crate::transient::TransientOracles })] +mod test {} -mod test { +mod value_test { use crate::test::helpers::test_environment::TestEnvironment; use crate::test::mocks::MockStruct; - use super::{delete, load, store, TransientArray}; - - global SLOT: Field = 1230; - global OTHER_SLOT: Field = 5670; - - #[test] - unconstrained fn empty_array() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array: TransientArray = TransientArray::at(SLOT); - assert_eq(array.len(), 0); - }); - } - - #[test(should_fail_with = "out of bounds")] - unconstrained fn empty_array_read() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - let _: Field = array.get(0); - }); - } - - #[test(should_fail_with = "is empty")] - unconstrained fn empty_array_pop() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - let _: Field = array.pop(); - }); - } - - #[test] - unconstrained fn array_push() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - array.push(5); - - assert_eq(array.len(), 1); - assert_eq(array.get(0), 5); - }); - } - - #[test(should_fail_with = "out of bounds")] - unconstrained fn read_past_len() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - array.push(5); - - let _ = array.get(1); - }); - } - - #[test] - unconstrained fn set_and_remove() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - array.push(7); - array.push(8); - array.push(9); - array.set(0, 99); - assert_eq(array.get(0), 99); - array.remove(1); - assert_eq(array.len(), 2); - assert_eq(array.get(0), 99); - assert_eq(array.get(1), 9); - }); - } - - #[test] - unconstrained fn array_pop() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - array.push(5); - array.push(10); - - let popped: Field = array.pop(); - assert_eq(popped, 10); - assert_eq(array.len(), 1); - assert_eq(array.get(0), 5); - }); - } - - #[test] - unconstrained fn array_remove_last() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - array.push(5); - array.remove(0); - assert_eq(array.len(), 0); - }); - } - - #[test] - unconstrained fn array_remove_some() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - - array.push(7); - array.push(8); - array.push(9); - - assert_eq(array.len(), 3); - - array.remove(1); - - assert_eq(array.len(), 2); - assert_eq(array.get(0), 7); - assert_eq(array.get(1), 9); - }); - } - - #[test] - unconstrained fn array_remove_all() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - - array.push(7); - array.push(8); - array.push(9); - - array.remove(1); - array.remove(1); - array.remove(0); - - assert_eq(array.len(), 0); - }); - } - - #[test] - unconstrained fn for_each_called_with_all_elements() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - - array.push(4); - array.push(5); - array.push(6); - - let called_with = &mut BoundedVec::<(u32, Field), 3>::new(); - array.for_each(|index, value| { called_with.push((index, value)); }); - - assert_eq(called_with.len(), 3); - assert(called_with.any(|(index, value)| (index == 0) & (value == 4))); - assert(called_with.any(|(index, value)| (index == 1) & (value == 5))); - assert(called_with.any(|(index, value)| (index == 2) & (value == 6))); - }); - } - - #[test] - unconstrained fn for_each_remove_some() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - - array.push(4); - array.push(5); - array.push(6); - - array.for_each(|index, _| { - if index == 1 { - array.remove(index); - } - }); - - assert_eq(array.len(), 2); - assert_eq(array.get(0), 4); - assert_eq(array.get(1), 6); - }); - } - - #[test] - unconstrained fn for_each_remove_all() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array = TransientArray::at(SLOT); - - array.push(4); - array.push(5); - array.push(6); - - array.for_each(|index, _| { array.remove(index); }); + use super::{TransientArray, TransientValue}; - assert_eq(array.len(), 0); - }); - } + global SLOT: Field = 9870; #[test] - unconstrained fn different_slots_are_isolated() { + unconstrained fn read_empty_returns_none() { let env = TestEnvironment::new(); env.utility_context(|_| { - let array_a = TransientArray::at(SLOT); - let array_b = TransientArray::at(OTHER_SLOT); - - array_a.push(10); - array_a.push(20); - array_b.push(99); - - assert_eq(array_a.len(), 2); - assert_eq(array_a.get(0), 10); - assert_eq(array_a.get(1), 20); - - assert_eq(array_b.len(), 1); - assert_eq(array_b.get(0), 99); + let value: TransientValue = TransientValue::at(SLOT); + assert_eq(value.read(), Option::none()); }); } #[test] - unconstrained fn works_with_multi_field_type() { + unconstrained fn store_and_read_roundtrips() { let env = TestEnvironment::new(); env.utility_context(|_| { - let array: TransientArray = TransientArray::at(SLOT); - - let a = MockStruct::new(5, 6); - let b = MockStruct::new(7, 8); - array.push(a); - array.push(b); - - assert_eq(array.len(), 2); - assert_eq(array.get(0), a); - assert_eq(array.get(1), b); - - let popped: MockStruct = array.pop(); - assert_eq(popped, b); - assert_eq(array.len(), 1); - }); - } - - #[test] - unconstrained fn clear_returns_self() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array: TransientArray = TransientArray::at(SLOT).clear(); - assert_eq(array.len(), 0); - - array.push(42); - assert_eq(array.len(), 1); - assert_eq(array.get(0), 42); - }); - } - - #[test] - unconstrained fn clear_wipes_previous_data() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let array: TransientArray = TransientArray::at(SLOT); - array.push(1); - array.push(2); - array.push(3); - assert_eq(array.len(), 3); - - let fresh: TransientArray = TransientArray::at(SLOT).clear(); - assert_eq(fresh.len(), 0); - fresh.push(4); - assert_eq(fresh.get(0), 4); - }); - } - - #[test] - unconstrained fn store_and_load() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - store(SLOT, 42); - assert_eq(load(SLOT), Option::some(42)); - }); - } - - #[test] - unconstrained fn load_empty_returns_none() { - let env = TestEnvironment::new(); - env.utility_context(|_| { - let value: Option = load(SLOT); - assert_eq(value, Option::none()); + let value: TransientValue = TransientValue::at(SLOT); + value.store(42); + assert_eq(value.read(), Option::some(42)); }); } @@ -454,9 +143,10 @@ mod test { unconstrained fn store_overwrites() { let env = TestEnvironment::new(); env.utility_context(|_| { - store(SLOT, 1); - store(SLOT, 2); - assert_eq(load(SLOT), Option::some(2)); + let value: TransientValue = TransientValue::at(SLOT); + value.store(1); + value.store(2); + assert_eq(value.read(), Option::some(2)); }); } @@ -464,10 +154,10 @@ mod test { unconstrained fn delete_removes_value() { let env = TestEnvironment::new(); env.utility_context(|_| { - store(SLOT, 42); - delete(SLOT); - let value: Option = load(SLOT); - assert_eq(value, Option::none()); + let value: TransientValue = TransientValue::at(SLOT); + value.store(42); + value.delete(); + assert_eq(value.read(), Option::none()); }); } @@ -475,19 +165,20 @@ mod test { unconstrained fn delete_empty_is_noop() { let env = TestEnvironment::new(); env.utility_context(|_| { - delete(SLOT); - let value: Option = load(SLOT); - assert_eq(value, Option::none()); + let value: TransientValue = TransientValue::at(SLOT); + value.delete(); + assert_eq(value.read(), Option::none()); }); } #[test] - unconstrained fn store_and_load_multi_field_type() { + unconstrained fn works_with_multi_field_type() { let env = TestEnvironment::new(); env.utility_context(|_| { - let value = MockStruct::new(5, 6); - store(SLOT, value); - assert_eq(load(SLOT), Option::some(value)); + let value: TransientValue = TransientValue::at(SLOT); + let stored = MockStruct::new(5, 6); + value.store(stored); + assert_eq(value.read(), Option::some(stored)); }); } @@ -495,7 +186,9 @@ mod test { unconstrained fn stored_value_is_visible_as_a_length_one_array() { let env = TestEnvironment::new(); env.utility_context(|_| { - store(SLOT, 42); + let value: TransientValue = TransientValue::at(SLOT); + value.store(42); + let array: TransientArray = TransientArray::at(SLOT); assert_eq(array.len(), 1); assert_eq(array.get(0), 42); diff --git a/noir-projects/aztec-nr/aztec/src/unconstrained_array/mod.nr b/noir-projects/aztec-nr/aztec/src/unconstrained_array/mod.nr new file mode 100644 index 000000000000..139d80bb42da --- /dev/null +++ b/noir-projects/aztec-nr/aztec/src/unconstrained_array/mod.nr @@ -0,0 +1,212 @@ +pub(crate) mod test_helpers; +pub(crate) mod test_suite; + +use crate::oracle::random::random; +use crate::protocol::traits::{Deserialize, Serialize}; + +/// Oracle backend for an [`UnconstrainedArray`]: the set of PXE-side operations that implement its storage. +/// +/// Each implementor routes these operations to a distinct family of oracles, and the oracle family determines the +/// array's lifetime and visibility (e.g. [`EphemeralArray`](crate::ephemeral::EphemeralArray) arrays live for one +/// contract call frame, while [`TransientArray`](crate::transient::TransientArray) arrays are shared across all frames +/// of the same contract within one top-level PXE call). +pub(crate) trait ArrayOracles { + /// Returns the number of elements in the array at `slot`. + unconstrained fn len_oracle(slot: Field) -> u32; + + /// Appends a serialized element to the array at `slot` and returns the new length. + unconstrained fn push_oracle(slot: Field, values: [Field; N]) -> u32; + + /// Removes and returns the last serialized element of the array at `slot`. Implementors must panic if the + /// array is empty. + unconstrained fn pop_oracle(slot: Field) -> [Field; N]; + + /// Returns the serialized element at the given index of the array at `slot`. Implementors must panic if `index` + /// is out of bounds. + unconstrained fn get_oracle(slot: Field, index: u32) -> [Field; N]; + + /// Overwrites the serialized element at the given index of the array at `slot`. Implementors must panic if + /// `index` is out of bounds. + unconstrained fn set_oracle(slot: Field, index: u32, values: [Field; N]); + + /// Removes the element at the given index of the array at `slot`, shifting subsequent elements backward. + /// Implementors must panic if `index` is out of bounds. + unconstrained fn remove_oracle(slot: Field, index: u32); + + /// Removes all elements from the array at `slot`. + unconstrained fn clear_oracle(slot: Field); +} + +/// A dynamically sized array backed by PXE-side in-memory storage via an [`ArrayOracles`] backend. +/// +/// Arrays are identified by a slot, and each logical operation (push, pop, get, etc.) is a single oracle call. The +/// `Oracle` backend determines the array's lifetime and visibility; contracts should not use this type directly but +/// rather one of its aliases: [`EphemeralArray`](crate::ephemeral::EphemeralArray) (scoped to a single contract call +/// frame) or [`TransientArray`](crate::transient::TransientArray) (shared across all frames of the same contract +/// within one top-level PXE call). +pub struct UnconstrainedArray { + pub(crate) slot: Field, +} + +impl UnconstrainedArray +where + Oracle: ArrayOracles, +{ + /// Returns a handle to the array at the given slot, which may already contain data (e.g. populated by an oracle + /// or by an earlier frame, depending on the backend's visibility). + pub unconstrained fn at(slot: Field) -> Self { + Self { slot } + } + + /// Returns an empty array at the given slot, clearing any pre-existing data. + /// + /// For backends whose arrays are visible beyond a single call frame (e.g. transient arrays), this wipes data + /// other frames of the same contract may have written at the slot. + pub unconstrained fn empty_at(slot: Field) -> Self { + Self::at(slot).clear() + } + + /// Returns an empty array at a fresh, randomly allocated slot. + /// + /// Use this when the caller does not need a specific slot: the random slot is isolated from every other array of + /// the same backend with overwhelming probability. Prefer [`UnconstrainedArray::empty_at`] when the slot must be a + /// known value (e.g. one shared with an oracle or another call frame). + pub unconstrained fn empty() -> Self { + Self::at(random()) + } + + /// Returns the number of elements stored in the array. + pub unconstrained fn len(self) -> u32 { + Oracle::len_oracle(self.slot) + } + + /// Stores a value at the end of the array. + pub unconstrained fn push(self, value: T) + where + T: Serialize, + { + let serialized = value.serialize(); + let _ = Oracle::push_oracle(self.slot, serialized); + } + + /// Removes and returns the last element. Implementors are required to panic if the array is empty. + pub unconstrained fn pop(self) -> T + where + T: Deserialize, + { + let serialized = Oracle::pop_oracle(self.slot); + Deserialize::deserialize(serialized) + } + + /// Retrieves the value stored at `index`. Implementors are required to panic if the index is out of bounds. + pub unconstrained fn get(self, index: u32) -> T + where + T: Deserialize, + { + let serialized = Oracle::get_oracle(self.slot, index); + Deserialize::deserialize(serialized) + } + + /// Overwrites the value stored at `index`. Implementors are required to panic if the index is out of bounds. + pub unconstrained fn set(self, index: u32, value: T) + where + T: Serialize, + { + let serialized = value.serialize(); + Oracle::set_oracle(self.slot, index, serialized); + } + + /// Removes the element at `index`, shifting subsequent elements backward. Implementors are required to panic if + /// the index is out of bounds. + pub unconstrained fn remove(self, index: u32) { + Oracle::remove_oracle(self.slot, index); + } + + /// Removes all elements from the array and returns self for chaining. + pub unconstrained fn clear(self) -> Self { + Oracle::clear_oracle(self.slot); + self + } + + /// Calls a function on each element of the array. + /// + /// The function `f` is called once with each array value and its corresponding index. Iteration proceeds + /// backwards so that it is safe to remove the current element (and only the current element) inside the + /// callback. + /// + /// It is **not** safe to push new elements from inside the callback. + pub unconstrained fn for_each(self, f: unconstrained fn[Env](u32, T) -> ()) + where + T: Deserialize, + { + let mut i = self.len(); + while i > 0 { + i -= 1; + f(i, self.get(i)); + } + } + + /// Applies `f` to every element and collects the results into a fresh array. + pub unconstrained fn map(self, f: unconstrained fn[Env](T) -> U) -> UnconstrainedArray + where + T: Deserialize, + U: Serialize, + { + let dest: UnconstrainedArray = UnconstrainedArray::empty(); + let n = self.len(); + for i in 0..n { + dest.push(f(self.get(i))); + } + dest + } + + /// Collects every element satisfying the predicate `f` into a fresh array. + pub unconstrained fn filter(self, f: unconstrained fn[Env](T) -> bool) -> Self + where + T: Serialize + Deserialize, + { + let dest: Self = UnconstrainedArray::empty(); + let n = self.len(); + for i in 0..n { + let value = self.get(i); + if f(value) { + dest.push(value); + } + } + dest + } + + /// Returns `true` if at least one element satisfies the predicate `f`. + pub unconstrained fn any(self, f: unconstrained fn[Env](T) -> bool) -> bool + where + T: Serialize + Deserialize, + { + self.filter(f).len() != 0 + } + + /// Returns `true` if every element satisfies the predicate `f` (vacuously `true` for an empty array). + pub unconstrained fn all(self, f: unconstrained fn[Env](T) -> bool) -> bool + where + T: Serialize + Deserialize, + { + self.filter(f).len() == self.len() + } + + /// Returns the first element satisfying the predicate `f`, or `Option::none` if none do. + pub unconstrained fn find(self, f: unconstrained fn[Env](T) -> bool) -> Option + where + T: Deserialize, + { + let n = self.len(); + let mut result: Option = Option::none(); + let mut i = 0; + while (i < n) & result.is_none() { + let value = self.get(i); + if f(value) { + result = Option::some(value); + } + i += 1; + } + result + } +} diff --git a/noir-projects/aztec-nr/aztec/src/unconstrained_array/test_helpers.nr b/noir-projects/aztec-nr/aztec/src/unconstrained_array/test_helpers.nr new file mode 100644 index 000000000000..cc7c96e753fd --- /dev/null +++ b/noir-projects/aztec-nr/aztec/src/unconstrained_array/test_helpers.nr @@ -0,0 +1,646 @@ +//! Shared test suite for [`UnconstrainedArray`] backends. +//! +//! Every function in this module must be an unconstrained test helper, generic over the [`ArrayOracles`] backend. +//! The [`unconstrained_array_tests`](crate::unconstrained_array::test_suite::unconstrained_array_tests) macro turns +//! each helper into a `#[test]` for every backend. +//! +//! To add a test, write a new unconstrained helper here. If it is expected to fail, mark it with +//! [`should_fail_test`](crate::unconstrained_array::test_suite::should_fail_test) and the expected message; otherwise +//! the generated test expects it to pass. + +use crate::test::helpers::test_environment::TestEnvironment; +use crate::test::mocks::MockStruct; +use crate::unconstrained_array::{ArrayOracles, UnconstrainedArray}; + +pub(crate) global SLOT: Field = 1230; +pub(crate) global OTHER_SLOT: Field = 5670; + +pub(crate) unconstrained fn empty_array() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + assert_eq(array.len(), 0); + }); +} + +#[crate::unconstrained_array::test_suite::should_fail_test(quote { "out of bounds" })] +pub(crate) unconstrained fn should_fail_empty_array_read() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + let _: Field = array.get(0); + }); +} + +#[crate::unconstrained_array::test_suite::should_fail_test(quote { "is empty" })] +pub(crate) unconstrained fn should_fail_empty_array_pop() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + let _: Field = array.pop(); + }); +} + +pub(crate) unconstrained fn array_push() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(5); + + assert_eq(array.len(), 1); + assert_eq(array.get(0), 5); + }); +} + +#[crate::unconstrained_array::test_suite::should_fail_test(quote { "out of bounds" })] +pub(crate) unconstrained fn should_fail_read_past_len() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(5); + + let _ = array.get(1); + }); +} + +pub(crate) unconstrained fn array_pop() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(5); + array.push(10); + + let popped: Field = array.pop(); + assert_eq(popped, 10); + assert_eq(array.len(), 1); + assert_eq(array.get(0), 5); + }); +} + +pub(crate) unconstrained fn array_set() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(5); + array.set(0, 99); + assert_eq(array.get(0), 99); + }); +} + +pub(crate) unconstrained fn array_remove_last() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(5); + array.remove(0); + assert_eq(array.len(), 0); + }); +} + +pub(crate) unconstrained fn array_remove_some() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + + array.push(7); + array.push(8); + array.push(9); + + assert_eq(array.len(), 3); + + array.remove(1); + + assert_eq(array.len(), 2); + assert_eq(array.get(0), 7); + assert_eq(array.get(1), 9); + }); +} + +pub(crate) unconstrained fn array_remove_all() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + + array.push(7); + array.push(8); + array.push(9); + + array.remove(1); + array.remove(1); + array.remove(0); + + assert_eq(array.len(), 0); + }); +} + +pub(crate) unconstrained fn for_each_called_with_all_elements() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + + array.push(4); + array.push(5); + array.push(6); + + let called_with = &mut BoundedVec::<(u32, Field), 3>::new(); + array.for_each(|index, value| { called_with.push((index, value)); }); + + assert_eq(called_with.len(), 3); + assert(called_with.any(|(index, value)| (index == 0) & (value == 4))); + assert(called_with.any(|(index, value)| (index == 1) & (value == 5))); + assert(called_with.any(|(index, value)| (index == 2) & (value == 6))); + }); +} + +pub(crate) unconstrained fn for_each_remove_some() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + + array.push(4); + array.push(5); + array.push(6); + + array.for_each(|index, _| { + if index == 1 { + array.remove(index); + } + }); + + assert_eq(array.len(), 2); + assert_eq(array.get(0), 4); + assert_eq(array.get(1), 6); + }); +} + +pub(crate) unconstrained fn for_each_remove_all() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + + array.push(4); + array.push(5); + array.push(6); + + array.for_each(|index, _| { array.remove(index); }); + + assert_eq(array.len(), 0); + }); +} + +pub(crate) unconstrained fn different_slots_are_isolated() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array_a: UnconstrainedArray = UnconstrainedArray::at(SLOT); + let array_b: UnconstrainedArray = UnconstrainedArray::at(OTHER_SLOT); + + array_a.push(10); + array_a.push(20); + array_b.push(99); + + assert_eq(array_a.len(), 2); + assert_eq(array_a.get(0), 10); + assert_eq(array_a.get(1), 20); + + assert_eq(array_b.len(), 1); + assert_eq(array_b.get(0), 99); + }); +} + +pub(crate) unconstrained fn works_with_multi_field_type() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + + let a = MockStruct::new(5, 6); + let b = MockStruct::new(7, 8); + array.push(a); + array.push(b); + + assert_eq(array.len(), 2); + assert_eq(array.get(0), a); + assert_eq(array.get(1), b); + + let popped: MockStruct = array.pop(); + assert_eq(popped, b); + assert_eq(array.len(), 1); + }); +} + +pub(crate) unconstrained fn clear_returns_self() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT).clear(); + assert_eq(array.len(), 0); + + array.push(42); + assert_eq(array.len(), 1); + assert_eq(array.get(0), 42); + }); +} + +pub(crate) unconstrained fn clear_wipes_previous_data() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(1); + array.push(2); + array.push(3); + assert_eq(array.len(), 3); + + // Clear the same slot, previous data should be gone. + let fresh: UnconstrainedArray = UnconstrainedArray::at(SLOT).clear(); + assert_eq(fresh.len(), 0); + fresh.push(4); + assert_eq(fresh.get(0), 4); + }); +} + +pub(crate) unconstrained fn empty_allocates_distinct_slots() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let a: UnconstrainedArray = UnconstrainedArray::empty(); + let b: UnconstrainedArray = UnconstrainedArray::empty(); + + assert(a.slot != b.slot, "empty() should allocate a fresh slot each time"); + + a.push(1); + b.push(2); + assert_eq(a.len(), 1); + assert_eq(a.get(0), 1); + assert_eq(b.len(), 1); + assert_eq(b.get(0), 2); + }); +} + +pub(crate) unconstrained fn map_transforms_each_element() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let source: UnconstrainedArray = UnconstrainedArray::at(SLOT); + source.push(1); + source.push(2); + source.push(3); + + let doubled: UnconstrainedArray = source.map(|x| x * 2); + + assert_eq(doubled.len(), 3); + assert_eq(doubled.get(0), 2); + assert_eq(doubled.get(1), 4); + assert_eq(doubled.get(2), 6); + + // The source array is left untouched. + assert_eq(source.len(), 3); + assert_eq(source.get(0), 1); + }); +} + +pub(crate) unconstrained fn map_empty_source_gives_empty_dest() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let source: UnconstrainedArray = UnconstrainedArray::at(SLOT); + let mapped: UnconstrainedArray = source.map(|x| x * 2); + assert_eq(mapped.len(), 0); + }); +} + +pub(crate) unconstrained fn map_to_different_type() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let source: UnconstrainedArray = UnconstrainedArray::at(SLOT); + source.push(5); + source.push(7); + + let structs: UnconstrainedArray = source.map(|x| MockStruct::new(x, x + 1)); + + assert_eq(structs.len(), 2); + assert_eq(structs.get(0), MockStruct::new(5, 6)); + assert_eq(structs.get(1), MockStruct::new(7, 8)); + }); +} + +pub(crate) unconstrained fn map_results_are_isolated() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let source: UnconstrainedArray = UnconstrainedArray::at(SLOT); + source.push(1); + source.push(2); + source.push(3); + + // Each map allocates its own fresh slot, so two maps of the same source do not clobber each other. + let doubled: UnconstrainedArray = source.map(|x| x * 2); + let tripled: UnconstrainedArray = source.map(|x| x * 3); + + assert(doubled.slot != tripled.slot, "each map should land in a distinct slot"); + assert_eq(doubled.get(0), 2); + assert_eq(doubled.get(2), 6); + assert_eq(tripled.get(0), 3); + assert_eq(tripled.get(2), 9); + }); +} + +pub(crate) unconstrained fn filter_keeps_matching_elements_in_order() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let source: UnconstrainedArray = UnconstrainedArray::at(SLOT); + source.push(1); + source.push(2); + source.push(3); + source.push(2); + source.push(5); + + let kept: UnconstrainedArray = source.filter(|x| x != 2); + + assert_eq(kept.len(), 3); + assert_eq(kept.get(0), 1); + assert_eq(kept.get(1), 3); + assert_eq(kept.get(2), 5); + + // The source array is left untouched. + assert_eq(source.len(), 5); + assert_eq(source.get(1), 2); + }); +} + +pub(crate) unconstrained fn filter_empty_source_gives_empty_dest() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let source: UnconstrainedArray = UnconstrainedArray::at(SLOT); + let kept: UnconstrainedArray = source.filter(|_| true); + assert_eq(kept.len(), 0); + }); +} + +pub(crate) unconstrained fn filter_none_match_gives_empty_dest() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let source: UnconstrainedArray = UnconstrainedArray::at(SLOT); + source.push(1); + source.push(2); + source.push(3); + + let kept: UnconstrainedArray = source.filter(|_| false); + assert_eq(kept.len(), 0); + }); +} + +pub(crate) unconstrained fn filter_works_with_multi_field_type() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let source: UnconstrainedArray = UnconstrainedArray::at(SLOT); + source.push(MockStruct::new(1, 10)); + source.push(MockStruct::new(2, 20)); + source.push(MockStruct::new(3, 30)); + + let kept: UnconstrainedArray = source.filter(|s: MockStruct| s.a != 2); + + assert_eq(kept.len(), 2); + assert_eq(kept.get(0), MockStruct::new(1, 10)); + assert_eq(kept.get(1), MockStruct::new(3, 30)); + }); +} + +pub(crate) unconstrained fn filter_results_are_isolated() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let source: UnconstrainedArray = UnconstrainedArray::at(SLOT); + source.push(1); + source.push(2); + source.push(3); + source.push(4); + + // Each filter allocates its own fresh slot, so two filters of the same source do not clobber each other. + let odds: UnconstrainedArray = source.filter(|x| (x != 2) & (x != 4)); + let evens: UnconstrainedArray = source.filter(|x| (x != 1) & (x != 3)); + + assert(odds.slot != evens.slot, "each filter should land in a distinct slot"); + assert_eq(odds.len(), 2); + assert_eq(odds.get(0), 1); + assert_eq(odds.get(1), 3); + assert_eq(evens.len(), 2); + assert_eq(evens.get(0), 2); + assert_eq(evens.get(1), 4); + }); +} + +pub(crate) unconstrained fn any_is_true_when_an_element_matches() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(1); + array.push(2); + array.push(3); + + assert(array.any(|x| x == 2)); + }); +} + +pub(crate) unconstrained fn any_is_false_when_no_element_matches() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(1); + array.push(2); + array.push(3); + + assert(!array.any(|x| x == 9)); + }); +} + +pub(crate) unconstrained fn any_on_empty_array_is_false() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + assert(!array.any(|_| true)); + }); +} + +pub(crate) unconstrained fn all_is_true_when_every_element_matches() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(1); + array.push(2); + array.push(3); + + assert(array.all(|x| x != 0)); + }); +} + +pub(crate) unconstrained fn all_is_false_when_one_element_fails() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(1); + array.push(2); + array.push(3); + + assert(!array.all(|x| x != 2)); + }); +} + +pub(crate) unconstrained fn all_on_empty_array_is_true() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + assert(array.all(|_| false)); + }); +} + +pub(crate) unconstrained fn find_returns_first_matching_element() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(1); + array.push(2); + array.push(3); + array.push(4); + + // Both 2 and 4 match; find must return the first one in order. + let found = array.find(|x| (x == 2) | (x == 4)); + assert(found.is_some()); + assert_eq(found.unwrap(), 2); + }); +} + +pub(crate) unconstrained fn find_returns_none_when_no_element_matches() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(1); + array.push(2); + array.push(3); + + assert(array.find(|x| x == 9).is_none()); + }); +} + +pub(crate) unconstrained fn find_on_empty_array_is_none() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + assert(array.find(|_| true).is_none()); + }); +} + +pub(crate) unconstrained fn empty_at_wipes_previous_data() +where + Oracle: ArrayOracles, +{ + let env = TestEnvironment::new(); + env.utility_context(|_| { + let array: UnconstrainedArray = UnconstrainedArray::at(SLOT); + array.push(1); + assert_eq(array.len(), 1); + + let fresh: UnconstrainedArray = UnconstrainedArray::empty_at(SLOT); + assert_eq(fresh.len(), 0); + }); +} diff --git a/noir-projects/aztec-nr/aztec/src/unconstrained_array/test_suite.nr b/noir-projects/aztec-nr/aztec/src/unconstrained_array/test_suite.nr new file mode 100644 index 000000000000..b4f1eea93f87 --- /dev/null +++ b/noir-projects/aztec-nr/aztec/src/unconstrained_array/test_suite.nr @@ -0,0 +1,46 @@ +//! Generates a shared [`UnconstrainedArray`](crate::unconstrained_array::UnconstrainedArray) +//! test suite from the helpers in [`test_helpers`](crate::unconstrained_array::test_helpers). + +use crate::utils::cmap::CHashMap; + +/// Maps test helpers marked with [`should_fail_test`] to their expected failure message. +pub(crate) comptime mut global EXPECTED_FAILURES: CHashMap = CHashMap::new(); + +/// Marks a test helper as expected to fail with the given message, passed as a quoted string literal: +/// `#[should_fail_test(quote { "out of bounds" })]`. The tests generated from the helper get +/// `#[test(should_fail_with = ...)]` instead of plain `#[test]`. +pub(crate) comptime fn should_fail_test(f: FunctionDefinition, message: Quoted) { + EXPECTED_FAILURES.insert(f, message); +} + +/// Generates a shared [`UnconstrainedArray`](crate::unconstrained_array::UnconstrainedArray) test suite against the +/// given backend. Apply to a test module, passing the backend's oracle as a quoted path: +/// `#[unconstrained_array_tests(quote { crate::ephemeral::EphemeralOracles })]`. +pub(crate) comptime fn unconstrained_array_tests(_m: Module, oracle: Quoted) -> Quoted { + let helpers = quote { crate::unconstrained_array::test_helpers }.as_module().unwrap(); + let mut tests = quote {}; + for f in helpers.functions() { + let name = f.name(); + if f.is_unconstrained() { + let fail = EXPECTED_FAILURES.get(f); + let attr = if fail.is_some() { + let msg = fail.unwrap(); + quote { #[test(should_fail_with = $msg)] } + } else { + quote { #[test] } + }; + tests = quote { + $tests + $attr + unconstrained fn $name() { + crate::unconstrained_array::test_helpers::$name::<$oracle>(); + } + }; + } else { + panic( + f"unexpected non-unconstrained function `{name}` in test_helpers: every test helper must be unconstrained", + ) + } + } + tests +} diff --git a/noir-projects/contract-snapshots/tests/snapshots/compile_failure/invalid_note/snapshots__stderr.snap b/noir-projects/contract-snapshots/tests/snapshots/compile_failure/invalid_note/snapshots__stderr.snap index b660c95f3ef1..9075cf02946b 100644 --- a/noir-projects/contract-snapshots/tests/snapshots/compile_failure/invalid_note/snapshots__stderr.snap +++ b/noir-projects/contract-snapshots/tests/snapshots/compile_failure/invalid_note/snapshots__stderr.snap @@ -15,8 +15,8 @@ error: InvalidNote has a packed length of 9 fields, which exceeds the maximum al at /noir-projects/aztec-nr/aztec/src/messages/discovery/mod.nr:: 3: sync_state_with_secrets at /noir-projects/aztec-nr/aztec/src/messages/discovery/mod.nr:: - 4: EphemeralArray::for_each - at /noir-projects/aztec-nr/aztec/src/ephemeral/mod.nr:: + 4: UnconstrainedArray::for_each + at /noir-projects/aztec-nr/aztec/src/unconstrained_array/mod.nr:: 5: sync_state_with_secrets at /noir-projects/aztec-nr/aztec/src/messages/discovery/mod.nr:: 6: process_message_ciphertext diff --git a/noir-projects/contract-snapshots/tests/snapshots/expand/amm_contract/snapshots__expanded.snap b/noir-projects/contract-snapshots/tests/snapshots/expand/amm_contract/snapshots__expanded.snap index c44ddd355c2f..da5b739dff4e 100644 --- a/noir-projects/contract-snapshots/tests/snapshots/expand/amm_contract/snapshots__expanded.snap +++ b/noir-projects/contract-snapshots/tests/snapshots/expand/amm_contract/snapshots__expanded.snap @@ -2,7 +2,6 @@ source: tests/snapshots.rs expression: stdout --- - use aztec::macros::aztec; use aztec::macros::aztec; @@ -833,7 +832,7 @@ pub contract AMM { unconstrained fn sync_state(scope: AztecAddress) { let address: AztecAddress = aztec::context::UtilityContext::new().this_address(); - aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, AztecAddress)>::none(), Option:: aztec::ephemeral::EphemeralArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); + aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, AztecAddress)>::none(), Option:: aztec::unconstrained_array::UnconstrainedArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); } pub struct offchain_receive_parameters { diff --git a/noir-projects/contract-snapshots/tests/snapshots/expand/avm_gadgets_test_contract/snapshots__expanded.snap b/noir-projects/contract-snapshots/tests/snapshots/expand/avm_gadgets_test_contract/snapshots__expanded.snap index c7f2843131a6..32db6054f5b6 100644 --- a/noir-projects/contract-snapshots/tests/snapshots/expand/avm_gadgets_test_contract/snapshots__expanded.snap +++ b/noir-projects/contract-snapshots/tests/snapshots/expand/avm_gadgets_test_contract/snapshots__expanded.snap @@ -2,7 +2,6 @@ source: tests/snapshots.rs expression: stdout --- - use aztec::macros::aztec; use aztec::macros::aztec; @@ -332,7 +331,7 @@ contract AvmGadgetsTest { unconstrained fn sync_state(scope: aztec::protocol::address::AztecAddress) { let address: aztec::protocol::address::AztecAddress = aztec::context::UtilityContext::new().this_address(); - aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, aztec::protocol::address::AztecAddress)>::none(), Option:: aztec::ephemeral::EphemeralArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); + aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, aztec::protocol::address::AztecAddress)>::none(), Option:: aztec::unconstrained_array::UnconstrainedArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); } pub struct offchain_receive_parameters { diff --git a/noir-projects/contract-snapshots/tests/snapshots/expand/avm_test_contract/snapshots__expanded.snap b/noir-projects/contract-snapshots/tests/snapshots/expand/avm_test_contract/snapshots__expanded.snap index cdfa8a6786b4..b04644838350 100644 --- a/noir-projects/contract-snapshots/tests/snapshots/expand/avm_test_contract/snapshots__expanded.snap +++ b/noir-projects/contract-snapshots/tests/snapshots/expand/avm_test_contract/snapshots__expanded.snap @@ -2,7 +2,6 @@ source: tests/snapshots.rs expression: stdout --- - use aztec::macros::aztec; use aztec::macros::aztec; @@ -1718,7 +1717,7 @@ pub contract AvmTest { unconstrained fn sync_state(scope: AztecAddress) { let address: AztecAddress = aztec::context::UtilityContext::new().this_address(); - aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, AztecAddress)>::none(), Option:: aztec::ephemeral::EphemeralArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); + aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, AztecAddress)>::none(), Option:: aztec::unconstrained_array::UnconstrainedArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); } pub struct offchain_receive_parameters { diff --git a/noir-projects/contract-snapshots/tests/snapshots/expand/public_fns_with_emit_repro_contract/snapshots__expanded.snap b/noir-projects/contract-snapshots/tests/snapshots/expand/public_fns_with_emit_repro_contract/snapshots__expanded.snap index 445edf3cc082..88d4e8960623 100644 --- a/noir-projects/contract-snapshots/tests/snapshots/expand/public_fns_with_emit_repro_contract/snapshots__expanded.snap +++ b/noir-projects/contract-snapshots/tests/snapshots/expand/public_fns_with_emit_repro_contract/snapshots__expanded.snap @@ -2,7 +2,6 @@ source: tests/snapshots.rs expression: stdout --- - use aztec::macros::aztec; use aztec::macros::aztec; @@ -308,7 +307,7 @@ pub contract PublicFnsWithEmitRepro { unconstrained fn sync_state(scope: aztec::protocol::address::AztecAddress) { let address: aztec::protocol::address::AztecAddress = aztec::context::UtilityContext::new().this_address(); - aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, aztec::protocol::address::AztecAddress)>::none(), Option:: aztec::ephemeral::EphemeralArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); + aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, aztec::protocol::address::AztecAddress)>::none(), Option:: aztec::unconstrained_array::UnconstrainedArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); } pub struct offchain_receive_parameters { diff --git a/noir-projects/contract-snapshots/tests/snapshots/expand/storage_proof_test_contract/snapshots__expanded.snap b/noir-projects/contract-snapshots/tests/snapshots/expand/storage_proof_test_contract/snapshots__expanded.snap index 0f8dae5d076b..ea4669fed6f3 100644 --- a/noir-projects/contract-snapshots/tests/snapshots/expand/storage_proof_test_contract/snapshots__expanded.snap +++ b/noir-projects/contract-snapshots/tests/snapshots/expand/storage_proof_test_contract/snapshots__expanded.snap @@ -2,7 +2,6 @@ source: tests/snapshots.rs expression: stdout --- - use aztec::macros::aztec; use aztec::macros::aztec; @@ -267,7 +266,7 @@ contract StorageProofTest { unconstrained fn sync_state(scope: AztecAddress) { let address: AztecAddress = aztec::context::UtilityContext::new().this_address(); - aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, AztecAddress)>::none(), Option:: aztec::ephemeral::EphemeralArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); + aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, AztecAddress)>::none(), Option:: aztec::unconstrained_array::UnconstrainedArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); } pub struct offchain_receive_parameters { diff --git a/noir-projects/contract-snapshots/tests/snapshots/expand/token_contract/snapshots__expanded.snap b/noir-projects/contract-snapshots/tests/snapshots/expand/token_contract/snapshots__expanded.snap index 61239ae02de8..8a9801c1afcd 100644 --- a/noir-projects/contract-snapshots/tests/snapshots/expand/token_contract/snapshots__expanded.snap +++ b/noir-projects/contract-snapshots/tests/snapshots/expand/token_contract/snapshots__expanded.snap @@ -2,7 +2,6 @@ source: tests/snapshots.rs expression: stdout --- - use aztec::macros::aztec; use aztec::macros::aztec; @@ -1043,7 +1042,7 @@ pub contract Token { unconstrained fn sync_state(scope: AztecAddress) { let address: AztecAddress = aztec::context::UtilityContext::new().this_address(); - aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, AztecAddress)>::none(), Option:: aztec::ephemeral::EphemeralArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); + aztec::messages::discovery::do_sync_state(address, _compute_note_hash, _compute_note_nullifier, Option::, aztec::messages::processing::MessageContext, AztecAddress)>::none(), Option:: aztec::unconstrained_array::UnconstrainedArray>::some(aztec::messages::processing::offchain::sync_inbox), scope); } pub struct offchain_receive_parameters { diff --git a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts index 057659691913..ee2f51b579ad 100644 --- a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts +++ b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts @@ -27,6 +27,7 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import type { BlockSynchronizerConfig } from '../config/index.js'; import type { ContractSyncService } from '../contract_sync/contract_sync_service.js'; import { AnchorBlockStore } from '../storage/anchor_block_store/anchor_block_store.js'; +import { EntityStore } from '../storage/entity_store/entity_store.js'; import { NoteStore } from '../storage/note_store/note_store.js'; import { PrivateEventStore } from '../storage/private_event_store/private_event_store.js'; import { BlockSynchronizer } from './block_synchronizer.js'; @@ -44,6 +45,7 @@ describe('BlockSynchronizer', () => { let anchorBlockStore: AnchorBlockStore; let noteStore: NoteStore; let privateEventStore: PrivateEventStore; + let entityStore: EntityStore; let aztecNode: MockProxy; let getBlock: NodeGetBlockMock; let blockStream: MockProxy; @@ -62,6 +64,7 @@ describe('BlockSynchronizer', () => { anchorBlockStore, noteStore, privateEventStore, + entityStore, tipsStore, contractSyncService, config, @@ -110,6 +113,7 @@ describe('BlockSynchronizer', () => { anchorBlockStore = new AnchorBlockStore(store); noteStore = new NoteStore(store); privateEventStore = new PrivateEventStore(store); + entityStore = new EntityStore(store); contractSyncService = mock(); synchronizer = createSynchronizer(); }); @@ -274,6 +278,99 @@ describe('BlockSynchronizer', () => { expect(await privateEventStore.eventIdsAtBlock(9)).toEqual([eventId9.toString()]); }); + it('chain-pruned deletes entities and facts originating above the fork, keeps non-retractable ones', async () => { + const contract = await AztecAddress.random(); + const scope = await AztecAddress.random(); + const entityTypeId = Fr.random(); + const prunedEntityId = Fr.random(); + const survivingEntityId = Fr.random(); + const retractableFactType = Fr.random(); + const nonRetractableFactType = Fr.random(); + const jobId = 'entity-job'; + + // Block 5 is the fork point; block 10 is on the abandoned fork (above the fork). + const forkBlock = await L2Block.random(BlockNumber(5)); + const block5 = makeL2BlockId(forkBlock.number, (await forkBlock.hash()).toString()); + const orphanedOriginBlock = { blockNumber: 10, blockHash: Fr.random() }; + + // A retractable entity originating on the abandoned fork: the prune must delete it wholesale, taking even its + // non-retractable fact with it. + await entityStore.createEntity( + contract, + scope, + entityTypeId, + prunedEntityId, + [Fr.random()], + orphanedOriginBlock, + jobId, + ); + await entityStore.recordFact( + contract, + scope, + entityTypeId, + prunedEntityId, + nonRetractableFactType, + [Fr.random()], + undefined, + jobId, + ); + + // A non-retractable entity: the prune must keep it, deleting only its fact originating on the abandoned fork. + await entityStore.createEntity(contract, scope, entityTypeId, survivingEntityId, [Fr.random()], undefined, jobId); + await entityStore.recordFact( + contract, + scope, + entityTypeId, + survivingEntityId, + nonRetractableFactType, + [Fr.random()], + undefined, + jobId, + ); + await entityStore.recordFact( + contract, + scope, + entityTypeId, + survivingEntityId, + retractableFactType, + [], + orphanedOriginBlock, + jobId, + ); + await store.transactionAsync(() => entityStore.commit(jobId)); + + // Both entities and all their facts must be present before the prune. + expect(await entityStore.activeEntities(contract, scope, entityTypeId, jobId)).toHaveLength(2); + expect(await entityStore.getEntityFacts(contract, scope, entityTypeId, prunedEntityId, jobId)).toHaveLength(1); + expect(await entityStore.getEntityFacts(contract, scope, entityTypeId, survivingEntityId, jobId)).toHaveLength(2); + + // Set the anchor to block 10 so the prune guard passes (anchor is above the fork point). + const anchorBlock10 = await L2Block.random(BlockNumber(10)); + await anchorBlockStore.setHeader(anchorBlock10.header); + + // The node serves the fork-point block; it becomes the new anchor after the prune. + const forkResponse = await blockResponse(forkBlock); + getBlock.mockImplementation(param => + Promise.resolve(param instanceof BlockHash && param.equals(forkResponse.hash) ? forkResponse : undefined), + ); + + // Prune back to block 5, orphaning block 10 where the retractable records originate. + await synchronizer.handleBlockStreamEvent({ + type: 'chain-pruned', + block: block5, + checkpoint: makeL2CheckpointId(CheckpointNumber.ZERO, GENESIS_CHECKPOINT_HEADER_HASH.toString()), + }); + + // The retractable entity is gone wholesale; the non-retractable one keeps only its non-retractable fact. + const active = await entityStore.activeEntities(contract, scope, entityTypeId, jobId); + expect(active).toHaveLength(1); + expect(active[0].equals(survivingEntityId)).toBe(true); + expect(await entityStore.getEntityFacts(contract, scope, entityTypeId, prunedEntityId, jobId)).toHaveLength(0); + const remaining = await entityStore.getEntityFacts(contract, scope, entityTypeId, survivingEntityId, jobId); + expect(remaining).toHaveLength(1); + expect(remaining[0].factTypeId.equals(nonRetractableFactType)).toBe(true); + }); + it('notes below the fork survive and remain queryable after a prune', async () => { const contract = await AztecAddress.random(); const scope = await AztecAddress.random(); diff --git a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts index f0e0735f9a0d..13e138c157fc 100644 --- a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts +++ b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts @@ -10,6 +10,7 @@ import type { BlockHeader } from '@aztec/stdlib/tx'; import type { BlockSynchronizerConfig } from '../config/index.js'; import type { ContractSyncService } from '../contract_sync/contract_sync_service.js'; import type { AnchorBlockStore } from '../storage/anchor_block_store/index.js'; +import type { EntityStore } from '../storage/entity_store/entity_store.js'; import type { NoteStore } from '../storage/note_store/index.js'; import type { PrivateEventStore } from '../storage/private_event_store/private_event_store.js'; import { blockStreamSourceFromAztecNode } from './block_stream_source.js'; @@ -31,6 +32,7 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler { private anchorBlockStore: AnchorBlockStore, private noteStore: NoteStore, private privateEventStore: PrivateEventStore, + private entityStore: EntityStore, private l2TipsStore: L2TipsKVStore, private contractSyncService: ContractSyncService, private config: Partial = {}, @@ -131,6 +133,7 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler { await this.store.transactionAsync(async () => { await this.noteStore.rollback(event.block.number); await this.privateEventStore.rollback(event.block.number); + await this.entityStore.rollback(event.block.number); await this.updateAnchorBlockHeader(newAnchorBlockHeader); }); break; diff --git a/yarn-project/pxe/src/pxe.ts b/yarn-project/pxe/src/pxe.ts index 299b9f6b3cd5..16d39de314fc 100644 --- a/yarn-project/pxe/src/pxe.ts +++ b/yarn-project/pxe/src/pxe.ts @@ -265,6 +265,7 @@ export class PXE { capsuleStore, keyStore, l2TipsStore, + entityStore, } = openPxeStores(store, initialBlockHash); const contractSyncService = new ContractSyncService( node, @@ -280,6 +281,7 @@ export class PXE { anchorBlockStore, noteStore, privateEventStore, + entityStore, l2TipsStore, contractSyncService, config, @@ -293,6 +295,7 @@ export class PXE { recipientTaggingStore, privateEventStore, noteStore, + entityStore, contractSyncService, ]); diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/EntityStore.json b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/EntityStore.json new file mode 100644 index 000000000000..6cf28f012500 --- /dev/null +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/EntityStore.json @@ -0,0 +1,54 @@ +{ + "entities": [ + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000aa", + "value": "00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000aa00000001000000000000000000000000000000000000000000000000000000000000000500000001000000060000000000000000000000000000000000000000000000000000000000000002" + }, + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb", + "value": "00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000bb00000001000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000" + } + ], + "entities_by_block": [ + { + "key": "num:6", + "value": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000aa" + } + ], + "entities_by_scope": [ + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007", + "value": "utf8:0x00000000000000000000000000000000000000000000000000000000000000aa" + }, + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007", + "value": "utf8:0x00000000000000000000000000000000000000000000000000000000000000bb" + } + ], + "facts": [ + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000001:0x00a802572c436574f713d43f1852859f4496d694de0e5b17b89cf983439e6143", + "value": "00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000bb000000000000000000000000000000000000000000000000000000000000000100000001000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000000000" + }, + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000002:0x00df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b811", + "value": "00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000bb00000000000000000000000000000000000000000000000000000000000000020000000000000001000000050000000000000000000000000000000000000000000000000000000000000001" + } + ], + "facts_by_entity": [ + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb", + "value": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000001:0x00a802572c436574f713d43f1852859f4496d694de0e5b17b89cf983439e6143" + }, + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb", + "value": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000002:0x00df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b811" + } + ], + "facts_by_block": [ + { + "key": "num:5", + "value": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000002:0x00df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b811" + } + ] +} diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json index 6c031f74812f..041e31309581 100644 --- a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json @@ -29,6 +29,18 @@ "name": "contracts_instances", "kind": "map" }, + { + "name": "entities", + "kind": "map" + }, + { + "name": "entities_by_block", + "kind": "multimap" + }, + { + "name": "entities_by_scope", + "kind": "multimap" + }, { "name": "events_by_block_number", "kind": "multimap" @@ -37,6 +49,18 @@ "name": "events_by_contract_selector", "kind": "multimap" }, + { + "name": "facts", + "kind": "map" + }, + { + "name": "facts_by_block", + "kind": "multimap" + }, + { + "name": "facts_by_entity", + "kind": "multimap" + }, { "name": "header", "kind": "singleton" diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts b/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts index 9a1e54a2a2cf..3f60f89ed5a8 100644 --- a/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts @@ -39,6 +39,7 @@ import { AddressStore } from '../address_store/address_store.js'; import { AnchorBlockStore } from '../anchor_block_store/index.js'; import { CapsuleStore } from '../capsule_store/capsule_store.js'; import { ContractStore } from '../contract_store/contract_store.js'; +import { EntityStore } from '../entity_store/entity_store.js'; import { NoteStore } from '../note_store/note_store.js'; import { PrivateEventStore } from '../private_event_store/private_event_store.js'; import { RecipientTaggingStore, SenderAddressBookStore, SenderTaggingStore } from '../tagging_store/index.js'; @@ -214,6 +215,51 @@ export const SCHEMA_TESTS: readonly SchemaTest[] = [ }), }, + { + name: 'EntityStore', + writeToStore: async kvStore => { + const entityStore = new EntityStore(kvStore); + const jobId = 'fixture-job'; + const contract = AztecAddress.fromBigInt(100n); + const scope = AztecAddress.fromBigInt(1n); + const entityTypeId = new Fr(7n); + const corrA = new Fr(0xaan); + const corrB = new Fr(0xbbn); + // Retractable entity (origin block 6): the entity and all its facts are pruned on a reorg above block 6. + await entityStore.createEntity( + contract, + scope, + entityTypeId, + corrA, + [new Fr(5n)], + { blockNumber: 6, blockHash: new Fr(2n) }, + jobId, + ); + // Non-retractable entity carrying a payload, with a non-retractable and a retractable fact. + await entityStore.createEntity(contract, scope, entityTypeId, corrB, [new Fr(8n)], undefined, jobId); + await entityStore.recordFact(contract, scope, entityTypeId, corrB, new Fr(1n), [new Fr(9n)], undefined, jobId); + await entityStore.recordFact( + contract, + scope, + entityTypeId, + corrB, + new Fr(2n), + [], + { blockNumber: 5, blockHash: new Fr(1n) }, + jobId, + ); + await kvStore.transactionAsync(() => entityStore.commit(jobId)); + }, + snapshotStore: async kvStore => ({ + entities: await snapshotMap(kvStore.openMap('entities')), + entities_by_block: await snapshotMap(kvStore.openMultiMap('entities_by_block')), + entities_by_scope: await snapshotMap(kvStore.openMultiMap('entities_by_scope')), + facts: await snapshotMap(kvStore.openMap('facts')), + facts_by_entity: await snapshotMap(kvStore.openMultiMap('facts_by_entity')), + facts_by_block: await snapshotMap(kvStore.openMultiMap('facts_by_block')), + }), + }, + { name: 'KeyStore', writeToStore: async kvStore => { diff --git a/yarn-project/pxe/src/storage/entity_store/entity_keys.ts b/yarn-project/pxe/src/storage/entity_store/entity_keys.ts new file mode 100644 index 000000000000..5ba542884f6a --- /dev/null +++ b/yarn-project/pxe/src/storage/entity_store/entity_keys.ts @@ -0,0 +1,33 @@ +import type { Fr } from '@aztec/foundation/curves/bn254'; +import type { AztecAddress } from '@aztec/stdlib/aztec-address'; + +/** The block a retractable entity or fact originates from. */ +export type OriginBlock = { blockNumber: number; blockHash: Fr }; + +/** The contract+scope+entityType+entityId coordinates shared by facts and entity records. */ +export type EntityCoords = { + contractAddress: AztecAddress; + scope: AztecAddress; + entityTypeId: Fr; + entityId: Fr; +}; + +/** Key that groups all entities of the same type within a contract+scope. */ +export function scopeKey(contract: AztecAddress, scope: AztecAddress, entityTypeId: Fr): string { + return `${contract}:${scope}:${entityTypeId}`; +} + +/** Key that uniquely identifies a single entity (all its facts share this key prefix). */ +export function entityKey(contract: AztecAddress, scope: AztecAddress, entityTypeId: Fr, entityId: Fr): string { + return `${scopeKey(contract, scope, entityTypeId)}:${entityId}`; +} + +/** Key that groups all entities of the same type within a contract+scope. */ +export function scopeKeyOf(coords: EntityCoords): string { + return scopeKey(coords.contractAddress, coords.scope, coords.entityTypeId); +} + +/** Key that uniquely identifies a single entity (all its facts share this key prefix). */ +export function entityKeyOf(coords: EntityCoords): string { + return entityKey(coords.contractAddress, coords.scope, coords.entityTypeId, coords.entityId); +} diff --git a/yarn-project/pxe/src/storage/entity_store/entity_store.test.ts b/yarn-project/pxe/src/storage/entity_store/entity_store.test.ts new file mode 100644 index 000000000000..f39a336a60ac --- /dev/null +++ b/yarn-project/pxe/src/storage/entity_store/entity_store.test.ts @@ -0,0 +1,344 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import type { AztecAsyncKVStore } from '@aztec/kv-store'; +import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import { EntityStore } from './entity_store.js'; + +describe('EntityStore', () => { + const contract = AztecAddress.fromBigInt(100n); + const scope = AztecAddress.fromBigInt(1n); + const ENTITY = new Fr(7n); + const RECEIVED = new Fr(1n); + const PROCESSED = new Fr(2n); + const corrA = new Fr(0xaan); + const corrB = new Fr(0xbbn); + const JOB = 'fact-store-test-job'; + + let kv: AztecAsyncKVStore; + let store: EntityStore; + + beforeEach(async () => { + kv = await openTmpStore('fact-store-test'); + store = new EntityStore(kv); + }); + afterEach(async () => { + await kv.close(); + }); + + it('records facts and loads an entity fact set after commit', async () => { + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + await store.recordFact( + contract, + scope, + ENTITY, + corrA, + PROCESSED, + [], + { blockNumber: 5, blockHash: new Fr(1n) }, + JOB, + ); + await kv.transactionAsync(() => store.commit(JOB)); + + const facts = await store.getEntityFacts(contract, scope, ENTITY, corrA, JOB); + const factTypes = facts.map(f => f.factTypeId.toBigInt()).sort(); + expect(factTypes).toEqual([RECEIVED.toBigInt(), PROCESSED.toBigInt()].sort()); + }); + + it('dedups identical (entity, factType, payload) records idempotently', async () => { + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const facts = await store.getEntityFacts(contract, scope, ENTITY, corrA, JOB); + expect(facts).toHaveLength(1); + }); + + it('enumerates active entities (created and not terminated) for a scope', async () => { + await store.createEntity(contract, scope, ENTITY, corrA, [], undefined, JOB); + await store.createEntity(contract, scope, ENTITY, corrB, [], undefined, JOB); + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + await store.recordFact(contract, scope, ENTITY, corrB, RECEIVED, [new Fr(8n)], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const active = await store.activeEntities(contract, scope, ENTITY, JOB); + expect(active.map(c => c.toBigInt()).sort()).toEqual([corrA.toBigInt(), corrB.toBigInt()].sort()); + }); + + it('lists an entity as active even when it has zero facts', async () => { + await store.createEntity(contract, scope, ENTITY, corrA, [new Fr(5n)], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + expect((await store.activeEntities(contract, scope, ENTITY, JOB)).map(c => c.toBigInt())).toEqual([ + corrA.toBigInt(), + ]); + }); + + it("reflects a job's own staged createEntity before commit (read-your-writes)", async () => { + await store.createEntity(contract, scope, ENTITY, corrA, [new Fr(5n)], undefined, JOB); + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + + // Same job, before commit: the staged entity is active and its fact is visible. + const facts = await store.getEntityFacts(contract, scope, ENTITY, corrA, JOB); + expect(facts.map(f => f.factTypeId.toBigInt())).toEqual([RECEIVED.toBigInt()]); + expect((await store.activeEntities(contract, scope, ENTITY, JOB)).map(c => c.toBigInt())).toEqual([ + corrA.toBigInt(), + ]); + + // A different job does not see the uncommitted write. + expect(await store.getEntityFacts(contract, scope, ENTITY, corrA, 'other-job')).toHaveLength(0); + expect(await store.activeEntities(contract, scope, ENTITY, 'other-job')).toHaveLength(0); + }); + + it('getEntity returns the payload and both facts of an entity with facts', async () => { + await store.createEntity(contract, scope, ENTITY, corrA, [new Fr(5n), new Fr(6n)], undefined, JOB); + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + await store.recordFact( + contract, + scope, + ENTITY, + corrA, + PROCESSED, + [], + { blockNumber: 5, blockHash: new Fr(1n) }, + JOB, + ); + await kv.transactionAsync(() => store.commit(JOB)); + + const { payload, facts } = await store.getEntity(contract, scope, ENTITY, corrA, JOB); + expect(payload.map(f => f.toBigInt())).toEqual([5n, 6n]); + expect(facts.map(f => f.factTypeId.toBigInt()).sort()).toEqual([RECEIVED.toBigInt(), PROCESSED.toBigInt()].sort()); + }); + + it('getEntity returns the payload and empty facts for an entity with zero facts', async () => { + await store.createEntity(contract, scope, ENTITY, corrA, [new Fr(5n)], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const { payload, facts } = await store.getEntity(contract, scope, ENTITY, corrA, JOB); + expect(payload.map(f => f.toBigInt())).toEqual([5n]); + expect(facts).toHaveLength(0); + }); + + it('getEntity returns an empty payload when no entity record exists', async () => { + const { payload, facts } = await store.getEntity(contract, scope, ENTITY, corrA, JOB); + expect(payload).toEqual([]); + expect(facts).toHaveLength(0); + }); + + it("getEntity reflects a job's own staged createEntity before commit (read-your-writes)", async () => { + await store.createEntity(contract, scope, ENTITY, corrA, [new Fr(5n)], undefined, JOB); + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + + const { payload, facts } = await store.getEntity(contract, scope, ENTITY, corrA, JOB); + expect(payload.map(f => f.toBigInt())).toEqual([5n]); + expect(facts.map(f => f.factTypeId.toBigInt())).toEqual([RECEIVED.toBigInt()]); + + // A different job does not see the uncommitted write. + expect((await store.getEntity(contract, scope, ENTITY, corrA, 'other-job')).payload).toEqual([]); + }); + + it('hides an entity from its own job after a staged terminate, even over committed facts', async () => { + await store.createEntity(contract, scope, ENTITY, corrA, [new Fr(5n)], undefined, JOB); + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + // A later job stages a terminate; within that job the entity reads as gone, while other jobs still see it + // committed until the terminate commits. + const TERM = 'terminate-job'; + await store.terminateEntity(contract, scope, ENTITY, corrA, TERM); + + expect(await store.getEntityFacts(contract, scope, ENTITY, corrA, TERM)).toHaveLength(0); + expect((await store.getEntity(contract, scope, ENTITY, corrA, TERM)).payload).toEqual([]); + expect(await store.activeEntities(contract, scope, ENTITY, TERM)).toHaveLength(0); + expect(await store.getEntityFacts(contract, scope, ENTITY, corrA, 'reader')).toHaveLength(1); + }); + + it('terminateEntity deletes the entity record, all its facts, and drops it from active enumeration', async () => { + await store.createEntity(contract, scope, ENTITY, corrA, [new Fr(5n)], undefined, JOB); + await store.createEntity(contract, scope, ENTITY, corrB, [], undefined, JOB); + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + await store.recordFact( + contract, + scope, + ENTITY, + corrA, + PROCESSED, + [], + { blockNumber: 5, blockHash: new Fr(1n) }, + JOB, + ); + await store.recordFact(contract, scope, ENTITY, corrB, RECEIVED, [new Fr(8n)], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const TERM = 'terminate-job'; + await store.terminateEntity(contract, scope, ENTITY, corrA, TERM); + await kv.transactionAsync(() => store.commit(TERM)); + + expect(await store.getEntityFacts(contract, scope, ENTITY, corrA, JOB)).toHaveLength(0); + expect((await store.getEntity(contract, scope, ENTITY, corrA, JOB)).payload).toEqual([]); + const active = await store.activeEntities(contract, scope, ENTITY, JOB); + expect(active.map(c => c.toBigInt())).toEqual([corrB.toBigInt()]); + // The neighbouring entity is untouched. + expect(await store.getEntityFacts(contract, scope, ENTITY, corrB, JOB)).toHaveLength(1); + }); + + it('rollback deletes a retractable entity wholesale (payload + every fact) above the target block', async () => { + // Retractable entity originating at block 6, owning one fact without an origin block and one with. + await store.createEntity( + contract, + scope, + ENTITY, + corrA, + [new Fr(5n)], + { blockNumber: 6, blockHash: new Fr(1n) }, + JOB, + ); + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + await store.recordFact( + contract, + scope, + ENTITY, + corrA, + PROCESSED, + [], + { blockNumber: 7, blockHash: new Fr(2n) }, + JOB, + ); + await kv.transactionAsync(() => store.commit(JOB)); + + await kv.transactionAsync(() => store.rollback(5)); + + const { payload, facts } = await store.getEntity(contract, scope, ENTITY, corrA, JOB); + expect(payload).toEqual([]); + expect(facts).toHaveLength(0); + expect(await store.getEntityFacts(contract, scope, ENTITY, corrA, JOB)).toHaveLength(0); + expect(await store.activeEntities(contract, scope, ENTITY, JOB)).toHaveLength(0); + }); + + it('rollback keeps a non-retractable entity, pruning only its retractable facts', async () => { + // Non-retractable entity (no origin block) with a non-retractable fact + a fact originating at block 6. + await store.createEntity(contract, scope, ENTITY, corrA, [new Fr(5n)], undefined, JOB); + await store.recordFact(contract, scope, ENTITY, corrA, RECEIVED, [new Fr(9n)], undefined, JOB); + await store.recordFact( + contract, + scope, + ENTITY, + corrA, + PROCESSED, + [], + { blockNumber: 6, blockHash: new Fr(1n) }, + JOB, + ); + await kv.transactionAsync(() => store.commit(JOB)); + + await kv.transactionAsync(() => store.rollback(5)); + + const { payload, facts } = await store.getEntity(contract, scope, ENTITY, corrA, JOB); + expect(payload.map(f => f.toBigInt())).toEqual([5n]); + expect(facts.map(f => f.factTypeId.toBigInt())).toEqual([RECEIVED.toBigInt()]); // Processed pruned, Received kept + // Entity stays active because the entity record survives. + expect((await store.activeEntities(contract, scope, ENTITY, JOB)).map(c => c.toBigInt())).toEqual([ + corrA.toBigInt(), + ]); + }); + + it('rollback above all origin blocks is a no-op', async () => { + await store.createEntity( + contract, + scope, + ENTITY, + corrA, + [new Fr(5n)], + { blockNumber: 6, blockHash: new Fr(1n) }, + JOB, + ); + await store.recordFact( + contract, + scope, + ENTITY, + corrA, + PROCESSED, + [], + { blockNumber: 7, blockHash: new Fr(2n) }, + JOB, + ); + await kv.transactionAsync(() => store.commit(JOB)); + + await kv.transactionAsync(() => store.rollback(10)); + + const { payload, facts } = await store.getEntity(contract, scope, ENTITY, corrA, JOB); + expect(payload.map(f => f.toBigInt())).toEqual([5n]); + expect(facts.map(f => f.factTypeId.toBigInt())).toEqual([PROCESSED.toBigInt()]); + expect((await store.activeEntities(contract, scope, ENTITY, JOB)).map(c => c.toBigInt())).toEqual([ + corrA.toBigInt(), + ]); + }); + + it('re-creating an entity with a changed origin block clears the stale by-block index', async () => { + await store.createEntity( + contract, + scope, + ENTITY, + corrA, + [new Fr(5n)], + { blockNumber: 6, blockHash: new Fr(1n) }, + JOB, + ); + await kv.transactionAsync(() => store.commit(JOB)); + + // Re-create the same entity originating at a different block. + const JOB2 = 'recreate-job'; + await store.createEntity( + contract, + scope, + ENTITY, + corrA, + [new Fr(5n)], + { blockNumber: 8, blockHash: new Fr(2n) }, + JOB2, + ); + await kv.transactionAsync(() => store.commit(JOB2)); + + // Prune above block 5: the entity (origin block 8) is deleted exactly once. A stale block-6 index entry would make + // pass 1 visit it a second time and throw "Entity not found". + await expect(kv.transactionAsync(() => store.rollback(5))).resolves.not.toThrow(); + expect((await store.getEntity(contract, scope, ENTITY, corrA, JOB)).payload).toEqual([]); + expect(await store.activeEntities(contract, scope, ENTITY, JOB)).toHaveLength(0); + }); + + it('re-creating a retractable entity as non-retractable lets it survive a prune', async () => { + await store.createEntity( + contract, + scope, + ENTITY, + corrA, + [new Fr(5n)], + { blockNumber: 6, blockHash: new Fr(1n) }, + JOB, + ); + await kv.transactionAsync(() => store.commit(JOB)); + + // Re-create the same entity without an origin block: it is now non-retractable and must survive reorgs. + const JOB2 = 'recreate-job'; + await store.createEntity(contract, scope, ENTITY, corrA, [new Fr(5n)], undefined, JOB2); + await kv.transactionAsync(() => store.commit(JOB2)); + + await kv.transactionAsync(() => store.rollback(5)); + + // Survived: the stale block-6 index entry was cleared when the entity was re-created. + const { payload } = await store.getEntity(contract, scope, ENTITY, corrA, JOB); + expect(payload.map(f => f.toBigInt())).toEqual([5n]); + expect((await store.activeEntities(contract, scope, ENTITY, JOB)).map(c => c.toBigInt())).toEqual([ + corrA.toBigInt(), + ]); + }); + + it('rollback throws while a job has staged writes', async () => { + await store.createEntity(contract, scope, ENTITY, corrA, [], undefined, 'uncommitted-job'); + await expect(kv.transactionAsync(() => store.rollback(0))).rejects.toThrow( + 'PXE entity store rollback is not allowed while jobs are running', + ); + await store.discardStaged('uncommitted-job'); + await expect(kv.transactionAsync(() => store.rollback(0))).resolves.not.toThrow(); + }); +}); diff --git a/yarn-project/pxe/src/storage/entity_store/entity_store.ts b/yarn-project/pxe/src/storage/entity_store/entity_store.ts new file mode 100644 index 000000000000..ec98e0f4ca2b --- /dev/null +++ b/yarn-project/pxe/src/storage/entity_store/entity_store.ts @@ -0,0 +1,439 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import { createLogger } from '@aztec/foundation/log'; +import { Semaphore } from '@aztec/foundation/queue'; +import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap } from '@aztec/kv-store'; +import type { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import type { StagedStore } from '../../job_coordinator/job_coordinator.js'; +import { type OriginBlock, entityKey, entityKeyOf, scopeKey, scopeKeyOf } from './entity_keys.js'; +import { StoredEntity } from './stored_entity.js'; +import { StoredFact, factRowKeyOf } from './stored_fact.js'; + +type JobId = string; + +/** A pending mutation for a job: create an entity, record a fact, or terminate (delete) an entity. */ +type StagedOp = + | { kind: 'createEntity'; entity: StoredEntity } + | { kind: 'record'; fact: StoredFact } + | { kind: 'terminate'; contract: AztecAddress; scope: AztecAddress; entityTypeId: Fr; entityId: Fr }; + +/** + * Stores immutable facts about entities, grouped by contract, scope, entity type, and entity id. + * + * Append-only within a job commit. Retractable facts (those with an origin block) are deleted on block prune. + * Non-retractable facts (originBlock === undefined) survive reorgs as external inputs. Writes are staged per-job and + * flushed atomically on commit. + */ +export class EntityStore implements StagedStore { + readonly storeName: string = 'entity'; + + #store: AztecAsyncKVStore; + /** Primary entity records, keyed by entityKey; holds the entity payload and optional origin block. */ + #entities: AztecAsyncMap; + /** Index from blockNumber to entityKey, for delete-on-prune of retractable entities (those with an origin block). */ + #entitiesByBlock: AztecAsyncMultiMap; + /** Primary fact records, keyed by factRowKey (deduplication row key). */ + #facts: AztecAsyncMap; + /** Index from entityKey to factRowKey for efficient entity-level fold. */ + #factsByEntity: AztecAsyncMultiMap; + /** Index from scopeKey to entityId string for active-entity enumeration. */ + #entitiesByScope: AztecAsyncMultiMap; + /** Index from blockNumber to factRowKey, for delete-on-prune (retractable facts only). */ + #factsByBlock: AztecAsyncMultiMap; + + #opsForJob: Map; + #jobLocks: Map; + + logger = createLogger('entity_store'); + + constructor(store: AztecAsyncKVStore) { + this.#store = store; + this.#entities = store.openMap('entities'); + this.#entitiesByBlock = store.openMultiMap('entities_by_block'); + this.#facts = store.openMap('facts'); + this.#factsByEntity = store.openMultiMap('facts_by_entity'); + this.#entitiesByScope = store.openMultiMap('entities_by_scope'); + this.#factsByBlock = store.openMultiMap('facts_by_block'); + this.#opsForJob = new Map(); + this.#jobLocks = new Map(); + } + + /** + * Stages an entity record (with its own payload and optional origin block) under the given job. + * `originBlock === undefined` marks the entity non-retractable (it survives reorgs; only its own retractable + * facts are pruned); a defined origin block marks the whole entity retractable — on a prune above its block, the + * entity and all its facts are deleted. + * The entity becomes active once committed, independently of whether it owns any facts. + */ + createEntity( + contract: AztecAddress, + scope: AztecAddress, + entityTypeId: Fr, + entityId: Fr, + payload: Fr[], + originBlock: OriginBlock | undefined, + jobId: string, + ): Promise { + return this.#withJobLock(jobId, () => { + const entity = new StoredEntity(contract, scope, entityTypeId, entityId, payload, originBlock); + this.#opsFor(jobId).push({ kind: 'createEntity', entity }); + return Promise.resolve(); + }); + } + + /** + * Stages a fact for recording under the given job. `originBlock === undefined` marks the fact non-retractable (it + * survives reorgs); a defined origin block ties the fact to a specific block and it will be deleted on prune. + * Idempotent: duplicate (entity, factType, payload) tuples collapse to a single row via the dedup row key. + */ + recordFact( + contract: AztecAddress, + scope: AztecAddress, + entityTypeId: Fr, + entityId: Fr, + factTypeId: Fr, + payload: Fr[], + originBlock: OriginBlock | undefined, + jobId: string, + ): Promise { + return this.#withJobLock(jobId, () => { + const fact = new StoredFact(contract, scope, entityTypeId, entityId, factTypeId, payload, originBlock); + this.#opsFor(jobId).push({ kind: 'record', fact }); + return Promise.resolve(); + }); + } + + /** Permanently delete an entity (all its facts). Staged within the job; applied on commit. */ + terminateEntity( + contract: AztecAddress, + scope: AztecAddress, + entityTypeId: Fr, + entityId: Fr, + jobId: string, + ): Promise { + return this.#withJobLock(jobId, () => { + this.#opsFor(jobId).push({ kind: 'terminate', contract, scope, entityTypeId, entityId }); + return Promise.resolve(); + }); + } + + /** + * Returns the facts for one entity. + * + * @param contract - The contract address owning the entity. + * @param scope - The scope (recipient address) under which facts were recorded. + * @param entityTypeId - Discriminates entity kinds within a contract+scope. + * @param entityId - Identifies the specific entity instance. + * @param jobId - The job whose staged writes are layered over committed state. + */ + async getEntityFacts( + contract: AztecAddress, + scope: AztecAddress, + entityTypeId: Fr, + entityId: Fr, + jobId: string, + ): Promise { + const eKey = entityKey(contract, scope, entityTypeId, entityId); + const byRow = await this.#store.transactionAsync(() => this.#loadCommittedFacts(eKey)); + this.#replayFactOps(byRow, eKey, jobId); + return Array.from(byRow.values()); + } + + /** + * Returns one entity's payload together with its facts. + * + * The payload comes from the entity record (empty when no entity record exists); the facts come from the per-entity + * fact index. This job's staged ops are layered over committed state for read-your-writes: a createEntity sets the + * payload, a record adds/dedups a fact, a terminate clears both payload and facts. + * + * @param contract - The contract address owning the entity. + * @param scope - The scope (recipient address) under which the entity was created. + * @param entityTypeId - Discriminates entity kinds within a contract+scope. + * @param entityId - Identifies the specific entity instance. + * @param jobId - The job whose staged writes are layered over committed state. + */ + async getEntity( + contract: AztecAddress, + scope: AztecAddress, + entityTypeId: Fr, + entityId: Fr, + jobId: string, + ): Promise<{ payload: Fr[]; facts: StoredFact[] }> { + const eKey = entityKey(contract, scope, entityTypeId, entityId); + const { payload, byRow } = await this.#store.transactionAsync(async () => { + const entityBuf = await this.#entities.getAsync(eKey); + return { + payload: entityBuf ? StoredEntity.fromBuffer(entityBuf).payload : [], + byRow: await this.#loadCommittedFacts(eKey), + }; + }); + // Replay this job's staged ops in order over committed state. Order matters so a terminate-then-create sequence + // resolves to the re-created entity, and a terminate clears both payload and facts. + let currentPayload = payload; + for (const op of this.#stagedOps(jobId)) { + if (op.kind === 'createEntity') { + if (entityKeyOf(op.entity) === eKey) { + currentPayload = op.entity.payload; + } + } else if (op.kind === 'record') { + if (entityKeyOf(op.fact) === eKey) { + byRow.set(factRowKeyOf(op.fact), op.fact); + } + } else if (entityKey(op.contract, op.scope, op.entityTypeId, op.entityId) === eKey) { + currentPayload = []; + byRow.clear(); + } + } + return { payload: currentPayload, facts: Array.from(byRow.values()) }; + } + + /** + * Returns the entity ids of all active entities under (contract, scope, entityTypeId) — entities that have an + * entity record and have not been terminated. Entity presence is independent of whether the entity owns any facts. + */ + async activeEntities(contract: AztecAddress, scope: AztecAddress, entityTypeId: Fr, jobId: string): Promise { + const sKey = scopeKey(contract, scope, entityTypeId); + const active = await this.#store.transactionAsync(async () => { + const seen = new Set(); + const result = new Set(); + for await (const entityId of this.#entitiesByScope.getValuesAsync(sKey)) { + if (seen.has(entityId)) { + continue; + } + seen.add(entityId); + // Guard against stale index entries: once terminate/rollback delete an entity record, its entity id + // should be gone from #entitiesByScope, but we re-check the entity record exists so a missed index update + // can never surface a ghost entity as active. + if (await this.#entities.getAsync(`${sKey}:${entityId}`)) { + result.add(entityId); + } + } + return result; + }); + // Replay this job's staged ops in order: a createEntity activates the entity, a terminate deactivates it. + for (const op of this.#stagedOps(jobId)) { + if (op.kind === 'createEntity') { + if (scopeKeyOf(op.entity) === sKey) { + active.add(op.entity.entityId.toString()); + } + } else if (op.kind === 'terminate' && scopeKey(op.contract, op.scope, op.entityTypeId) === sKey) { + active.delete(op.entityId.toString()); + } + } + return Array.from(active, Fr.fromString); + } + + /** + * Commits all staged operations for the given job to persistent storage. + * + * Must be called inside a transaction owned by the caller (JobCoordinator wraps all commits in a single + * transactionAsync, and IndexedDB does not support nested transactions). Do not call #withJobLock here — awaiting + * the lock creates a microtask boundary that causes IndexedDB to auto-commit the outer transaction. + */ + async commit(jobId: string): Promise { + for (const op of this.#opsFor(jobId)) { + if (op.kind === 'createEntity') { + const entity = op.entity; + const eKey = entityKeyOf(entity); + // Re-creating an entity may change or drop its origin block; clear any stale by-block index entry from the + // record first, so a later prune can neither double-visit this entity (and throw) nor wrongly delete one that + // has since become non-retractable. + const priorBuf = await this.#entities.getAsync(eKey); + if (priorBuf) { + const prior = StoredEntity.fromBuffer(priorBuf); + if (prior.originBlock !== undefined) { + await this.#entitiesByBlock.deleteValue(prior.originBlock.blockNumber, eKey); + } + } + await this.#entities.set(eKey, entity.toBuffer()); + await this.#entitiesByScope.set(scopeKeyOf(entity), entity.entityId.toString()); + if (entity.originBlock !== undefined) { + await this.#entitiesByBlock.set(entity.originBlock.blockNumber, eKey); + } + } else if (op.kind === 'record') { + const fact = op.fact; + const rowKey = factRowKeyOf(fact); + await this.#facts.set(rowKey, fact.toBuffer()); + await this.#factsByEntity.set(entityKeyOf(fact), rowKey); + if (fact.originBlock !== undefined) { + await this.#factsByBlock.set(fact.originBlock.blockNumber, rowKey); + } + } else { + await this.#deleteEntity(op.contract, op.scope, op.entityTypeId, op.entityId); + } + } + this.#clearJobData(jobId); + } + + /** Discards all staged operations for the given job without persisting them. */ + discardStaged(jobId: string): Promise { + this.#clearJobData(jobId); + return Promise.resolve(); + } + + /** + * Delete-on-prune in two passes. Pass 1 deletes every retractable entity whose origin block is strictly above + * `toBlock` wholesale — its payload and every fact it owns, regardless of each fact's own flag. Pass 2 deletes any + * remaining retractable fact originating above `toBlock` whose entity survived pass 1. Non-retractable entities and + * facts are untouched (they never enter the by-block indexes). Must run inside a caller-owned transaction (the + * reorg path wraps it with the sibling stores' rollbacks; IndexedDB has no nested transactions). Throws if any job + * has uncommitted staged writes, since rolling back mid-job could re-introduce records originating from deleted + * blocks. + */ + async rollback(toBlock: number): Promise { + if (this.#opsForJob.size > 0) { + throw new Error('PXE entity store rollback is not allowed while jobs are running'); + } + + // Pass 1: delete retractable entities originating above toBlock wholesale. Snapshot before mutating so we never + // delete from the multimap we are iterating. + const orphanedEntities: string[] = []; + for await (const [, eKey] of this.#entitiesByBlock.entriesAsync({ start: toBlock + 1 })) { + orphanedEntities.push(eKey); + } + const deletedEntities = new Set(); + let removedEntities = 0; + for (const eKey of orphanedEntities) { + const buf = await this.#entities.getAsync(eKey); + if (!buf) { + // An #entitiesByBlock entry must always reference a live #entities row; a missing one means the indexes are + // corrupt, so fail loudly rather than leave a ghost entity behind. + throw new Error(`Entity not found for entityKey ${eKey}`); + } + const entity = StoredEntity.fromBuffer(buf); + await this.#deleteEntity(entity.contractAddress, entity.scope, entity.entityTypeId, entity.entityId); + deletedEntities.add(eKey); + removedEntities++; + } + + // Pass 2: delete remaining retractable facts originating above toBlock whose entity survived pass 1. Pass 1 already + // removed the facts_by_block rows of pruned entities, so any leftover row here belongs to a surviving entity. + const orphanedFacts: { block: number; rowKey: string }[] = []; + for await (const [block, rowKey] of this.#factsByBlock.entriesAsync({ start: toBlock + 1 })) { + orphanedFacts.push({ block, rowKey }); + } + let removedFacts = 0; + for (const { block, rowKey } of orphanedFacts) { + const buf = await this.#facts.getAsync(rowKey); + if (!buf) { + // A still-present by-block entry with no fact row means the indexes are corrupt (pass 1 removes both the row + // and the by-block entry for pruned entities), so fail loudly rather than leave a dangling index entry. + throw new Error(`Fact not found for rowKey ${rowKey}`); + } + const fact = StoredFact.fromBuffer(buf); + const eKey = entityKeyOf(fact); + // Belt-and-braces: a fact whose entity was pruned in pass 1 must not be re-processed here. With #deleteEntity + // clearing the by-block index this is unreachable, but the guard keeps pass 2 correct if that ever changes. + if (deletedEntities.has(eKey)) { + continue; + } + await this.#facts.delete(rowKey); + await this.#factsByBlock.deleteValue(block, rowKey); + await this.#factsByEntity.deleteValue(eKey, rowKey); + removedFacts++; + } + this.logger.verbose('rolled back entity store', { removedEntities, removedFacts, toBlock }); + } + + // ---- private helpers ---- + + /** Loads the committed facts for an entity keyed by their dedup row key. Caller may wrap in a transaction. */ + async #loadCommittedFacts(eKey: string): Promise> { + const rows = new Map(); + for await (const rowKey of this.#factsByEntity.getValuesAsync(eKey)) { + const buf = await this.#facts.getAsync(rowKey); + if (buf) { + rows.set(rowKey, StoredFact.fromBuffer(buf)); + } + } + return rows; + } + + /** + * Replays a job's staged ops over a committed fact map for read-your-writes: a record sets (dedups) a fact row, a + * terminate clears the entity's facts. Order matters so a terminate-then-record sequence resolves to the re-recorded + * fact. createEntity ops do not affect the fact set. + */ + #replayFactOps(byRow: Map, eKey: string, jobId: string): void { + for (const op of this.#stagedOps(jobId)) { + if (op.kind === 'record') { + if (entityKeyOf(op.fact) === eKey) { + byRow.set(factRowKeyOf(op.fact), op.fact); + } + } else if (op.kind === 'terminate' && entityKey(op.contract, op.scope, op.entityTypeId, op.entityId) === eKey) { + byRow.clear(); + } + } + } + + /** + * Deletes an entity wholesale from every index: its record, all its facts, and the scope/block index entries. + * Called during commit for 'terminate' ops and during pass 1 of rollback for pruned retractable entities. + */ + async #deleteEntity(contract: AztecAddress, scope: AztecAddress, entityTypeId: Fr, entityId: Fr): Promise { + const eKey = entityKey(contract, scope, entityTypeId, entityId); + const rowKeys: string[] = []; + for await (const rowKey of this.#factsByEntity.getValuesAsync(eKey)) { + rowKeys.push(rowKey); + } + for (const rowKey of rowKeys) { + const buf = await this.#facts.getAsync(rowKey); + if (!buf) { + // A #factsByEntity entry must always reference a live #facts row; a missing one means the indexes are + // corrupt, so fail loudly rather than silently skip cleanup. + throw new Error(`Fact not found for rowKey ${rowKey}`); + } + const fact = StoredFact.fromBuffer(buf); + await this.#facts.delete(rowKey); + await this.#factsByEntity.deleteValue(eKey, rowKey); + if (fact.originBlock !== undefined) { + await this.#factsByBlock.deleteValue(fact.originBlock.blockNumber, rowKey); + } + } + const entityBuf = await this.#entities.getAsync(eKey); + if (entityBuf) { + const entity = StoredEntity.fromBuffer(entityBuf); + await this.#entities.delete(eKey); + if (entity.originBlock !== undefined) { + await this.#entitiesByBlock.deleteValue(entity.originBlock.blockNumber, eKey); + } + } + await this.#entitiesByScope.deleteValue(scopeKey(contract, scope, entityTypeId), entityId.toString()); + } + + #opsFor(jobId: string): StagedOp[] { + let ops = this.#opsForJob.get(jobId); + if (ops === undefined) { + ops = []; + this.#opsForJob.set(jobId, ops); + } + return ops; + } + + /** + * Read-only view of a job's staged ops for read-your-writes. Unlike {@link #opsFor}, it never creates an entry, so a + * read for a job with no writes does not register a phantom in-flight job (which would trip the `rollback` guard). + */ + #stagedOps(jobId: string): StagedOp[] { + return this.#opsForJob.get(jobId) ?? []; + } + + #clearJobData(jobId: string) { + this.#opsForJob.delete(jobId); + this.#jobLocks.delete(jobId); + } + + async #withJobLock(jobId: string, fn: () => Promise): Promise { + let lock = this.#jobLocks.get(jobId); + if (!lock) { + lock = new Semaphore(1); + this.#jobLocks.set(jobId, lock); + } + await lock.acquire(); + try { + return await fn(); + } finally { + lock.release(); + } + } +} diff --git a/yarn-project/pxe/src/storage/entity_store/index.ts b/yarn-project/pxe/src/storage/entity_store/index.ts new file mode 100644 index 000000000000..8c58301b3bcc --- /dev/null +++ b/yarn-project/pxe/src/storage/entity_store/index.ts @@ -0,0 +1 @@ +export { EntityStore } from './entity_store.js'; diff --git a/yarn-project/pxe/src/storage/entity_store/stored_entity.test.ts b/yarn-project/pxe/src/storage/entity_store/stored_entity.test.ts new file mode 100644 index 000000000000..5901e1ad0322 --- /dev/null +++ b/yarn-project/pxe/src/storage/entity_store/stored_entity.test.ts @@ -0,0 +1,35 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import { StoredEntity } from './stored_entity.js'; + +describe('StoredEntity', () => { + const contract = AztecAddress.fromBigInt(100n); + const scope = AztecAddress.fromBigInt(1n); + const entityType = new Fr(7n); + const entityId = new Fr(42n); + + it('round-trips a retractable entity through buffer serialization', () => { + const entity = new StoredEntity(contract, scope, entityType, entityId, [new Fr(9n), new Fr(10n)], { + blockNumber: 12, + blockHash: new Fr(0xabcn), + }); + const back = StoredEntity.fromBuffer(entity.toBuffer()); + expect(back).toEqual(entity); + expect(back.isRetractable).toBe(true); + }); + + it('round-trips a non-retractable entity (no origin block)', () => { + const entity = new StoredEntity(contract, scope, entityType, entityId, [new Fr(9n)], undefined); + const back = StoredEntity.fromBuffer(entity.toBuffer()); + expect(back).toEqual(entity); + expect(back.isRetractable).toBe(false); + }); + + it('round-trips an entity with an empty payload', () => { + const entity = new StoredEntity(contract, scope, entityType, entityId, [], undefined); + const back = StoredEntity.fromBuffer(entity.toBuffer()); + expect(back).toEqual(entity); + expect(back.payload).toEqual([]); + }); +}); diff --git a/yarn-project/pxe/src/storage/entity_store/stored_entity.ts b/yarn-project/pxe/src/storage/entity_store/stored_entity.ts new file mode 100644 index 000000000000..0f8e9b0ecbd5 --- /dev/null +++ b/yarn-project/pxe/src/storage/entity_store/stored_entity.ts @@ -0,0 +1,57 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import type { OriginBlock } from './entity_keys.js'; + +/** + * The record for a single entity, with its own payload and optional origin block. `originBlock === undefined` marks + * the entity non-retractable (it survives reorgs; only its own retractable facts are pruned); an origin block marks + * the whole entity retractable — on a prune above its block, the entity payload and every fact it owns are deleted + * wholesale. + */ +export class StoredEntity { + constructor( + public readonly contractAddress: AztecAddress, + public readonly scope: AztecAddress, + public readonly entityTypeId: Fr, + public readonly entityId: Fr, + public readonly payload: Fr[], + public readonly originBlock: OriginBlock | undefined, + ) {} + + /** Whether the whole entity is deleted on block pruning (true) or survives reorgs (false). */ + get isRetractable(): boolean { + return this.originBlock !== undefined; + } + + toBuffer(): Buffer { + const originBlockTag = this.originBlock ? 1 : 0; + return serializeToBuffer( + this.contractAddress, + this.scope, + this.entityTypeId, + this.entityId, + this.payload.length, + ...this.payload, + originBlockTag, + this.originBlock ? this.originBlock.blockNumber : 0, + this.originBlock ? this.originBlock.blockHash : Fr.ZERO, + ); + } + + static fromBuffer(buffer: Buffer | BufferReader): StoredEntity { + const reader = BufferReader.asReader(buffer); + const contractAddress = reader.readObject(AztecAddress); + const scope = reader.readObject(AztecAddress); + const entityTypeId = reader.readObject(Fr); + const entityId = reader.readObject(Fr); + const payloadLen = reader.readNumber(); + const payload = reader.readArray(payloadLen, Fr); + const originBlockTag = reader.readNumber(); + const blockNumber = reader.readNumber(); + const blockHash = reader.readObject(Fr); + const originBlock = originBlockTag === 1 ? { blockNumber, blockHash } : undefined; + return new StoredEntity(contractAddress, scope, entityTypeId, entityId, [...payload], originBlock); + } +} diff --git a/yarn-project/pxe/src/storage/entity_store/stored_fact.test.ts b/yarn-project/pxe/src/storage/entity_store/stored_fact.test.ts new file mode 100644 index 000000000000..481db557186c --- /dev/null +++ b/yarn-project/pxe/src/storage/entity_store/stored_fact.test.ts @@ -0,0 +1,45 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import { entityKeyOf, scopeKeyOf } from './entity_keys.js'; +import { StoredFact, factRowKeyOf } from './stored_fact.js'; + +describe('StoredFact', () => { + const contract = AztecAddress.fromBigInt(100n); + const scope = AztecAddress.fromBigInt(1n); + const entityType = new Fr(7n); + const entityId = new Fr(42n); + const factType = new Fr(3n); + + it('round-trips a retractable fact through buffer serialization', () => { + const fact = new StoredFact(contract, scope, entityType, entityId, factType, [new Fr(9n), new Fr(10n)], { + blockNumber: 12, + blockHash: new Fr(0xabcn), + }); + const back = StoredFact.fromBuffer(fact.toBuffer()); + expect(back).toEqual(fact); + expect(back.isRetractable).toBe(true); + }); + + it('round-trips a non-retractable fact (no origin block)', () => { + const fact = new StoredFact(contract, scope, entityType, entityId, factType, [new Fr(9n)], undefined); + const back = StoredFact.fromBuffer(fact.toBuffer()); + expect(back).toEqual(fact); + expect(back.isRetractable).toBe(false); + }); + + it('derives stable composite keys', () => { + const fact = new StoredFact(contract, scope, entityType, entityId, factType, [new Fr(9n)], undefined); + expect(scopeKeyOf(fact)).toBe(`${contract}:${scope}:${entityType}`); + expect(entityKeyOf(fact)).toBe(`${contract}:${scope}:${entityType}:${entityId}`); + expect(factRowKeyOf(fact)).toBe(entityKeyOf(fact) + `:${factType}:${fact.payloadHash()}`); + }); + + it('gives distinct payload hashes for distinct payloads and equal for equal', () => { + const a = new StoredFact(contract, scope, entityType, entityId, factType, [new Fr(1n)], undefined); + const b = new StoredFact(contract, scope, entityType, entityId, factType, [new Fr(2n)], undefined); + const c = new StoredFact(contract, scope, entityType, entityId, factType, [new Fr(1n)], undefined); + expect(a.payloadHash()).not.toEqual(b.payloadHash()); + expect(a.payloadHash()).toEqual(c.payloadHash()); + }); +}); diff --git a/yarn-project/pxe/src/storage/entity_store/stored_fact.ts b/yarn-project/pxe/src/storage/entity_store/stored_fact.ts new file mode 100644 index 000000000000..198818a7db61 --- /dev/null +++ b/yarn-project/pxe/src/storage/entity_store/stored_fact.ts @@ -0,0 +1,69 @@ +import { sha256ToField } from '@aztec/foundation/crypto/sha256'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import { type OriginBlock, entityKeyOf } from './entity_keys.js'; + +/** + * A single immutable fact about an entity. `originBlock === undefined` marks the fact non-retractable (an external + * input that survives reorgs); an origin block marks it retractable (re-derivable, deleted when its block is pruned). + */ +export class StoredFact { + constructor( + public readonly contractAddress: AztecAddress, + public readonly scope: AztecAddress, + public readonly entityTypeId: Fr, + public readonly entityId: Fr, + public readonly factTypeId: Fr, + public readonly payload: Fr[], + public readonly originBlock: OriginBlock | undefined, + ) {} + + /** Whether this fact is deleted on block pruning (true) or survives reorgs (false). */ + get isRetractable(): boolean { + return this.originBlock !== undefined; + } + + /** Stable digest of the payload, used in the dedup row key (keeps the LMDB key bounded for large payloads). */ + payloadHash(): Fr { + return sha256ToField([this.payload.length, ...this.payload]); + } + + toBuffer(): Buffer { + const originBlockTag = this.originBlock ? 1 : 0; + return serializeToBuffer( + this.contractAddress, + this.scope, + this.entityTypeId, + this.entityId, + this.factTypeId, + this.payload.length, + ...this.payload, + originBlockTag, + this.originBlock ? this.originBlock.blockNumber : 0, + this.originBlock ? this.originBlock.blockHash : Fr.ZERO, + ); + } + + static fromBuffer(buffer: Buffer | BufferReader): StoredFact { + const reader = BufferReader.asReader(buffer); + const contractAddress = reader.readObject(AztecAddress); + const scope = reader.readObject(AztecAddress); + const entityTypeId = reader.readObject(Fr); + const entityId = reader.readObject(Fr); + const factTypeId = reader.readObject(Fr); + const payloadLen = reader.readNumber(); + const payload = reader.readArray(payloadLen, Fr); + const originBlockTag = reader.readNumber(); + const blockNumber = reader.readNumber(); + const blockHash = reader.readObject(Fr); + const originBlock = originBlockTag === 1 ? { blockNumber, blockHash } : undefined; + return new StoredFact(contractAddress, scope, entityTypeId, entityId, factTypeId, [...payload], originBlock); + } +} + +/** Dedup row key for a specific fact; incorporates a payload hash to bound key size for large payloads. */ +export function factRowKeyOf(fact: StoredFact): string { + return `${entityKeyOf(fact)}:${fact.factTypeId}:${fact.payloadHash()}`; +} diff --git a/yarn-project/pxe/src/storage/index.ts b/yarn-project/pxe/src/storage/index.ts index 47c5b840f7ee..c52e353eb6d1 100644 --- a/yarn-project/pxe/src/storage/index.ts +++ b/yarn-project/pxe/src/storage/index.ts @@ -2,6 +2,7 @@ export * from './address_store/index.js'; export * from './anchor_block_store/index.js'; export * from './capsule_store/index.js'; export * from './contract_store/index.js'; +export * from './entity_store/index.js'; export * from './note_store/index.js'; export * from './tagging_store/index.js'; export * from './metadata.js'; diff --git a/yarn-project/pxe/src/storage/open_pxe_stores.ts b/yarn-project/pxe/src/storage/open_pxe_stores.ts index 573b85a40984..45d6d131ae79 100644 --- a/yarn-project/pxe/src/storage/open_pxe_stores.ts +++ b/yarn-project/pxe/src/storage/open_pxe_stores.ts @@ -7,6 +7,7 @@ import { AddressStore } from './address_store/address_store.js'; import { AnchorBlockStore } from './anchor_block_store/anchor_block_store.js'; import { CapsuleStore } from './capsule_store/capsule_store.js'; import { ContractStore } from './contract_store/contract_store.js'; +import { EntityStore } from './entity_store/entity_store.js'; import { NoteStore } from './note_store/note_store.js'; import { PrivateEventStore } from './private_event_store/private_event_store.js'; import { RecipientTaggingStore, SenderAddressBookStore, SenderTaggingStore } from './tagging_store/index.js'; @@ -26,6 +27,7 @@ export type PxeStores = { capsuleStore: CapsuleStore; keyStore: KeyStore; l2TipsStore: L2TipsKVStore; + entityStore: EntityStore; }; /** @@ -45,5 +47,6 @@ export function openPxeStores(store: AztecAsyncKVStore, initialBlockHash: BlockH capsuleStore: new CapsuleStore(store), keyStore: new KeyStore(store), l2TipsStore: new L2TipsKVStore(store, 'pxe', initialBlockHash), + entityStore: new EntityStore(store), }; }