From ff5aeb237ab4bbed9e560cb5c76362f95d53a645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:08:08 -0300 Subject: [PATCH 1/8] fix(types): encode a beacon state without holding it twice libssz's container encoder writes each fixed-size field past the end of the buffer and then moves it into place, and its to_ssz reserves only the encoding's length. Fulu's state ends on a fixed-size field (proposer_lookahead) and so does gloas's (ptc_window), so encoding either regrows a buffer the size of the whole state to twice that for its last field, copying it. Encoding a 44 MB fulu state ended with an 88 MB buffer. BeaconState now reserves the encoding's length plus its largest fixed-size field (ssz_append_capacity) and appends into that, so to_ssz never regrows; the Beacon API's SSZ state response goes through it. The storage layer's encode_state_value, which runs on every import, appends behind its fork tag into one such buffer instead of copying a separately encoded state, which removed a second state-sized copy from the follower's per-slot memory spike. --- .../common/types/src/beacon/containers/mod.rs | 62 ++++++++++++++++++- crates/storage/src/state_codec.rs | 24 ++++++- 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index d1f926edf..fa633c08e 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -366,8 +366,49 @@ impl BeaconState { /// unlike the accessors that read a beacon field: this is the inverse of /// [`BeaconState::from_ssz`], which builds that variant, so refusing here /// would make decoding a state and re-encoding it panic. + /// + /// Appends into a buffer with [`BeaconState::ssz_append_capacity`] rather + /// than calling libssz's own `to_ssz`, which reserves only the encoding's + /// length and so holds a fulu or gloas state twice while encoding it (see + /// there). pub fn to_ssz(&self) -> Vec { - dispatch_state_including_lean!(self, |state| state.to_ssz()) + let mut buf = Vec::with_capacity(self.ssz_append_capacity()); + self.ssz_append(&mut buf); + buf + } + + /// The spare capacity [`BeaconState::ssz_append`] needs in `buf` for the + /// append never to regrow it: the encoding's own length plus the largest + /// fixed-size field any fork's state has. + /// + /// The headroom is for libssz's container encoder, which writes each + /// fixed-size field past the end of the buffer and then moves it into + /// place. A fixed-size field written after every variable-size one + /// (fulu's `proposer_lookahead`, gloas's `ptc_window`) therefore needs + /// room beyond the encoding's length, and without it a buffer the size of + /// the whole state is regrown to twice that, copying it, for the last few + /// hundred bytes. + pub fn ssz_append_capacity(&self) -> usize { + let largest_fixed_field = [ + ::fixed_size(), + ::fixed_size(), + ::fixed_size(), + ::fixed_size(), + ] + .into_iter() + .max() + .expect("the list is not empty"); + dispatch_state_including_lean!(self, |state| state.encoded_len()) + largest_fixed_field + } + + /// Appends [`BeaconState::to_ssz`]'s output to `buf`. + /// + /// For a caller that frames the encoding (the storage layer's fork tag): + /// a mainnet state encodes to hundreds of megabytes, and appending to a + /// buffer with [`BeaconState::ssz_append_capacity`] to spare holds one + /// copy of it, never two. + pub fn ssz_append(&self, buf: &mut Vec) { + dispatch_state_including_lean!(self, |state| state.ssz_append(buf)) } /// The state's merkle root, which a block's `state_root` must equal. @@ -2179,6 +2220,25 @@ mod tests { }) } + /// Fulu and gloas end on a fixed-size field, which is where libssz's + /// encoder writes past the encoding's length. + #[test] + fn ssz_append_never_regrows_a_buffer_with_its_capacity_to_spare() { + for state in [small_fulu_state(3), small_gloas_state(3)] { + let fork = state.fork_name(); + let expected = dispatch_state_including_lean!(&state, |state| state.to_ssz()); + let capacity = state.ssz_append_capacity(); + let mut buf = Vec::with_capacity(capacity); + state.ssz_append(&mut buf); + assert_eq!(buf, expected, "{fork}"); + assert_eq!(buf.capacity(), capacity, "{fork}: ssz_append regrew"); + + let encoded = state.to_ssz(); + assert_eq!(encoded, expected, "{fork}"); + assert_eq!(encoded.capacity(), capacity, "{fork}: to_ssz regrew"); + } + } + #[test] fn registry_accessors_read_and_write_elements() { let mut state = small_fulu_state(3); // 3 validators, balances 32 ETH diff --git a/crates/storage/src/state_codec.rs b/crates/storage/src/state_codec.rs index 3e0255d89..dc65b87b9 100644 --- a/crates/storage/src/state_codec.rs +++ b/crates/storage/src/state_codec.rs @@ -24,9 +24,13 @@ use ethlambda_types::{ /// [`ForkName::Lean`]'s selector is not a variant index, so the byte must go /// back through [`ForkName::from_selector`] rather than being cast. pub(crate) fn encode_state_value(state: &BeaconState) -> Vec { - let mut bytes = Vec::new(); + // Encoded straight into the tagged buffer, sized so the encoder never + // regrows it: the state writer runs this on every import, and either a + // separate `Vec` or a regrowth would hold two copies of a mainnet state + // at once. + let mut bytes = Vec::with_capacity(1 + state.ssz_append_capacity()); bytes.push(state.fork_name().selector()); - bytes.extend_from_slice(&state.to_ssz()); + state.ssz_append(&mut bytes); bytes } @@ -66,6 +70,22 @@ mod tests { assert_eq!(decode_state_value(&bytes), state); } + #[test] + fn a_beacon_state_value_is_its_tag_then_its_ssz_in_one_buffer() { + let electra = crate::beacon_state_delta::tests::electra_state_with_validators(64); + let state = BeaconState::Electra(electra); + let bytes = encode_state_value(&state); + + assert_eq!(bytes[0], ForkName::Electra.selector()); + assert_eq!(bytes[1..], state.to_ssz()); + assert_eq!( + bytes.capacity(), + 1 + state.ssz_append_capacity(), + "the buffer was regrown" + ); + assert_eq!(decode_state_value(&bytes), state); + } + #[test] fn the_selector_is_not_a_dense_index() { // ForkName::Lean is 255 so that beacon forks after fulu keep taking the From 06804a0cb64a66f5dfc8523aafa0b39142ad9358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:26:59 -0300 Subject: [PATCH 2/8] test(types): check the no-regrow capacity against every fork The test covered fulu and gloas only, so a later fork with a fixed-size field larger than any ssz_append_capacity counts would have lost the guarantee without failing anything. It now builds a small state of every fork, the hard case since little variable-size data follows each fixed field, and lives with the state-transition crate's test builders because ethlambda-types cannot build a state of every fork. --- .../src/beacon/helpers/test_state.rs | 32 +++++++++++++++++++ .../common/types/src/beacon/containers/mod.rs | 19 ----------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs index 373e12262..e51fdc80d 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs @@ -652,3 +652,35 @@ fn gloas_state(count: usize) -> gloas::BeaconState { .expect("the vector is built at its exact length"), } } + +#[cfg(test)] +mod tests { + use super::*; + + /// [`BeaconState::ssz_append_capacity`] for every fork. The test lives here + /// rather than beside that method in `ethlambda-types` because this module + /// is the one place a state of every fork is built. + /// + /// A small registry is the hard case. libssz's container encoder writes + /// each fixed-size field past the end of the buffer before moving it into + /// place, so a field needs room beyond the encoding's length unless enough + /// variable-size data still follows it, and a small state has little. A + /// fork that adds a fixed-size field larger than any the capacity counts + /// fails here rather than quietly regrowing a buffer the size of the state. + #[test] + fn ssz_append_never_regrows_a_buffer_with_its_capacity_to_spare() { + for fork in ForkName::ALL { + let state = with_validators_at(fork, 3); + let capacity = state.ssz_append_capacity(); + let mut buf = Vec::with_capacity(capacity); + state.ssz_append(&mut buf); + assert_eq!(buf.capacity(), capacity, "{fork}: ssz_append regrew"); + let decoded = BeaconState::from_ssz(fork, &buf).expect("the encoding decodes"); + assert_eq!(decoded, state, "{fork}"); + + let encoded = state.to_ssz(); + assert_eq!(encoded, buf, "{fork}"); + assert_eq!(encoded.capacity(), capacity, "{fork}: to_ssz regrew"); + } + } +} diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index fa633c08e..b96384c5c 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -2220,25 +2220,6 @@ mod tests { }) } - /// Fulu and gloas end on a fixed-size field, which is where libssz's - /// encoder writes past the encoding's length. - #[test] - fn ssz_append_never_regrows_a_buffer_with_its_capacity_to_spare() { - for state in [small_fulu_state(3), small_gloas_state(3)] { - let fork = state.fork_name(); - let expected = dispatch_state_including_lean!(&state, |state| state.to_ssz()); - let capacity = state.ssz_append_capacity(); - let mut buf = Vec::with_capacity(capacity); - state.ssz_append(&mut buf); - assert_eq!(buf, expected, "{fork}"); - assert_eq!(buf.capacity(), capacity, "{fork}: ssz_append regrew"); - - let encoded = state.to_ssz(); - assert_eq!(encoded, expected, "{fork}"); - assert_eq!(encoded.capacity(), capacity, "{fork}: to_ssz regrew"); - } - } - #[test] fn registry_accessors_read_and_write_elements() { let mut state = small_fulu_state(3); // 3 validators, balances 32 ETH From 556209330e59a71f0cabb12ce0c3263c90d1c74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:28:19 -0300 Subject: [PATCH 3/8] fix(types): have ssz_append reserve its own room Each caller had to remember to size the buffer with ssz_append_capacity, and to add its own prefix to it. One that forgot (vec![tag] and then ssz_append, or a reserve of encoded_len only) regrew a buffer the size of the state and held two copies of it, the very spike this change set out to remove. ssz_append now reserves the room past the buffer's current length itself, so a prefix is covered, and to_ssz and the state codec no longer compute it. --- .../src/beacon/helpers/test_state.rs | 15 ++++++---- .../common/types/src/beacon/containers/mod.rs | 28 +++++++++++-------- crates/storage/src/state_codec.rs | 10 +++---- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs index e51fdc80d..4198ac540 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs @@ -672,15 +672,18 @@ mod tests { for fork in ForkName::ALL { let state = with_validators_at(fork, 3); let capacity = state.ssz_append_capacity(); - let mut buf = Vec::with_capacity(capacity); - state.ssz_append(&mut buf); - assert_eq!(buf.capacity(), capacity, "{fork}: ssz_append regrew"); - let decoded = BeaconState::from_ssz(fork, &buf).expect("the encoding decodes"); - assert_eq!(decoded, state, "{fork}"); let encoded = state.to_ssz(); - assert_eq!(encoded, buf, "{fork}"); assert_eq!(encoded.capacity(), capacity, "{fork}: to_ssz regrew"); + let decoded = BeaconState::from_ssz(fork, &encoded).expect("the encoding decodes"); + assert_eq!(decoded, state, "{fork}"); + + // Appending after a prefix reserves past the prefix, not from the + // start of the buffer. + let mut framed = vec![0xAB]; + state.ssz_append(&mut framed); + assert_eq!(framed.capacity(), 1 + capacity, "{fork}: ssz_append regrew"); + assert_eq!(framed[1..], encoded, "{fork}"); } } } diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index b96384c5c..977aa5200 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -367,19 +367,20 @@ impl BeaconState { /// [`BeaconState::from_ssz`], which builds that variant, so refusing here /// would make decoding a state and re-encoding it panic. /// - /// Appends into a buffer with [`BeaconState::ssz_append_capacity`] rather - /// than calling libssz's own `to_ssz`, which reserves only the encoding's - /// length and so holds a fulu or gloas state twice while encoding it (see - /// there). + /// Goes through [`BeaconState::ssz_append`] rather than libssz's own + /// `to_ssz`, which reserves only the encoding's length and so holds a fulu + /// or gloas state twice while encoding it (see + /// [`BeaconState::ssz_append_capacity`]). pub fn to_ssz(&self) -> Vec { - let mut buf = Vec::with_capacity(self.ssz_append_capacity()); + let mut buf = Vec::new(); self.ssz_append(&mut buf); buf } - /// The spare capacity [`BeaconState::ssz_append`] needs in `buf` for the - /// append never to regrow it: the encoding's own length plus the largest - /// fixed-size field any fork's state has. + /// How much [`BeaconState::ssz_append`] reserves past the end of the + /// buffer it appends to, so the append never regrows it: the encoding's + /// own length plus the largest fixed-size field any fork's state has. + /// Public for tests that check that bound. /// /// The headroom is for libssz's container encoder, which writes each /// fixed-size field past the end of the buffer and then moves it into @@ -401,13 +402,16 @@ impl BeaconState { dispatch_state_including_lean!(self, |state| state.encoded_len()) + largest_fixed_field } - /// Appends [`BeaconState::to_ssz`]'s output to `buf`. + /// Appends [`BeaconState::to_ssz`]'s output to `buf`, after whatever it + /// already holds. /// /// For a caller that frames the encoding (the storage layer's fork tag): - /// a mainnet state encodes to hundreds of megabytes, and appending to a - /// buffer with [`BeaconState::ssz_append_capacity`] to spare holds one - /// copy of it, never two. + /// a mainnet state encodes to hundreds of megabytes, and the append holds + /// one copy of it, never two. It first reserves + /// [`BeaconState::ssz_append_capacity`] past `buf`'s length, so a caller + /// need not size `buf` itself. pub fn ssz_append(&self, buf: &mut Vec) { + buf.reserve_exact(self.ssz_append_capacity()); dispatch_state_including_lean!(self, |state| state.ssz_append(buf)) } diff --git a/crates/storage/src/state_codec.rs b/crates/storage/src/state_codec.rs index dc65b87b9..8278f6326 100644 --- a/crates/storage/src/state_codec.rs +++ b/crates/storage/src/state_codec.rs @@ -24,12 +24,10 @@ use ethlambda_types::{ /// [`ForkName::Lean`]'s selector is not a variant index, so the byte must go /// back through [`ForkName::from_selector`] rather than being cast. pub(crate) fn encode_state_value(state: &BeaconState) -> Vec { - // Encoded straight into the tagged buffer, sized so the encoder never - // regrows it: the state writer runs this on every import, and either a - // separate `Vec` or a regrowth would hold two copies of a mainnet state - // at once. - let mut bytes = Vec::with_capacity(1 + state.ssz_append_capacity()); - bytes.push(state.fork_name().selector()); + // Encoded straight into the tagged buffer: the state writer runs this on + // every import, and a separate `Vec` would hold two copies of a mainnet + // state at once. `ssz_append` reserves its own room past the tag. + let mut bytes = vec![state.fork_name().selector()]; state.ssz_append(&mut bytes); bytes } From 00620d7040f8ee326e486ac276bd3d6dfb03da8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:30:36 -0300 Subject: [PATCH 4/8] fix(types): size a lean state's encode headroom from its own fields Every variant reserved the largest fixed-size field of any beacon fork, 2 MiB on mainnet, including a lean state, whose widest fixed field is a block header. Each lean snapshot and to_ssz of a lean state allocated megabytes it never used, often more than the state itself. A lean state now reserves its own largest fixed field. The beacon forks keep one shared list, since every one of them has randao_mixes and a list per fork would save at most a kilobyte on minimal. The every-fork test now covers a lean genesis too; with no headroom for it, that case regrows. --- .../src/beacon/helpers/test_state.rs | 8 +++- .../common/types/src/beacon/containers/mod.rs | 43 ++++++++++++++----- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs index 4198ac540..d939fa20d 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs @@ -669,8 +669,12 @@ mod tests { /// fails here rather than quietly regrowing a buffer the size of the state. #[test] fn ssz_append_never_regrows_a_buffer_with_its_capacity_to_spare() { - for fork in ForkName::ALL { - let state = with_validators_at(fork, 3); + // A lean genesis with no validators: its fixed-size fields come before + // all of its variable-size ones, which hold almost nothing here. + let lean = BeaconState::Lean(ethlambda_types::state::State::from_genesis(7, vec![])); + let beacon = ForkName::ALL.map(|fork| with_validators_at(fork, 3)); + for state in beacon.into_iter().chain([lean]) { + let fork = state.fork_name(); let capacity = state.ssz_append_capacity(); let encoded = state.to_ssz(); diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 977aa5200..0fb340705 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -379,7 +379,7 @@ impl BeaconState { /// How much [`BeaconState::ssz_append`] reserves past the end of the /// buffer it appends to, so the append never regrows it: the encoding's - /// own length plus the largest fixed-size field any fork's state has. + /// own length plus the largest fixed-size field of this variant's state. /// Public for tests that check that bound. /// /// The headroom is for libssz's container encoder, which writes each @@ -390,16 +390,37 @@ impl BeaconState { /// the whole state is regrown to twice that, copying it, for the last few /// hundred bytes. pub fn ssz_append_capacity(&self) -> usize { - let largest_fixed_field = [ - ::fixed_size(), - ::fixed_size(), - ::fixed_size(), - ::fixed_size(), - ] - .into_iter() - .max() - .expect("the list is not empty"); - dispatch_state_including_lean!(self, |state| state.encoded_len()) + largest_fixed_field + dispatch_state_including_lean!(self, |state| state.encoded_len()) + + self.largest_fixed_field() + } + + /// The largest fixed-size field in this variant's state: how far past the + /// encoding's end libssz's encoder can write (see + /// [`BeaconState::ssz_append_capacity`]). + /// + /// The beacon forks share one list. Every one of them has `randao_mixes`, + /// the largest on mainnet, and the only field that beats it on any preset + /// is gloas's `ptc_window` on minimal, by about a kilobyte, so a list per + /// fork would save nothing worth keeping eight lists for. A lean state is + /// a different container with fields a few hundred bytes wide, and + /// reserving a beacon state's megabytes for it would often outweigh the + /// state itself. + fn largest_fixed_field(&self) -> usize { + use libssz::SszEncode; + let largest = |sizes: &[usize]| sizes.iter().copied().max().unwrap_or(0); + match self { + BeaconState::Lean(_) => largest(&[ + ::fixed_size(), + ::fixed_size(), + ::fixed_size(), + ]), + _ => largest(&[ + ::fixed_size(), + ::fixed_size(), + ::fixed_size(), + ::fixed_size(), + ]), + } } /// Appends [`BeaconState::to_ssz`]'s output to `buf`, after whatever it From 808dbd64cd43f0dae62a36d2bcf98a8d1b2a4867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:32:27 -0300 Subject: [PATCH 5/8] fix(storage): write a beacon snapshot without cloning it for the memo plan_beacon_state_write cloned the encoded target at every snapshot, one copy for the States row and one for the writer's snapshot memo, so a snapshot write still held a mainnet state twice, plus RocksDB's own copy in the batch. The plan now hands over one buffer and a flag. The writer puts it into the batch from a borrow, through a new StorageWriteBatch::put that takes slices (RocksDB copies the value into its batch either way), and then moves the same buffer into the memo. The floor is now the target plus the batch's copy until commit. --- crates/storage/src/api/traits.rs | 10 +++++ crates/storage/src/backend/rocksdb.rs | 10 +++++ crates/storage/src/backend/tests.rs | 18 ++++++++ crates/storage/src/state_writer.rs | 59 +++++++++++++++++---------- 4 files changed, 75 insertions(+), 22 deletions(-) diff --git a/crates/storage/src/api/traits.rs b/crates/storage/src/api/traits.rs index 35747be3c..e7f04643d 100644 --- a/crates/storage/src/api/traits.rs +++ b/crates/storage/src/api/traits.rs @@ -97,6 +97,16 @@ pub trait StorageWriteBatch: Send { /// Put multiple key-value pairs into a table. fn put_batch(&mut self, table: Table, batch: Vec<(Vec, Vec)>) -> Result<(), Error>; + /// Put one key-value pair into a table, from borrowed bytes. + /// + /// For a value the caller keeps after the write, so it need not clone one + /// only to hand it over: a beacon snapshot is hundreds of megabytes, and + /// RocksDB copies the value into its batch either way. The default goes + /// through [`put_batch`](Self::put_batch). + fn put(&mut self, table: Table, key: &[u8], value: &[u8]) -> Result<(), Error> { + self.put_batch(table, vec![(key.to_vec(), value.to_vec())]) + } + /// Delete multiple keys from a table. fn delete_batch(&mut self, table: Table, keys: Vec>) -> Result<(), Error>; diff --git a/crates/storage/src/backend/rocksdb.rs b/crates/storage/src/backend/rocksdb.rs index 4e51943c6..d74ad860a 100644 --- a/crates/storage/src/backend/rocksdb.rs +++ b/crates/storage/src/backend/rocksdb.rs @@ -219,6 +219,16 @@ impl StorageWriteBatch for RocksDBWriteBatch { Ok(()) } + fn put(&mut self, table: Table, key: &[u8], value: &[u8]) -> Result<(), Error> { + let cf = self + .db + .cf_handle(cf_name(table)) + .ok_or_else(|| format!("Column family {} not found", cf_name(table)))?; + // The batch copies the bytes, so the caller's buffer is never cloned. + self.batch.put_cf(&cf, key, value); + Ok(()) + } + fn delete_batch(&mut self, table: Table, keys: Vec>) -> Result<(), Error> { let cf = self .db diff --git a/crates/storage/src/backend/tests.rs b/crates/storage/src/backend/tests.rs index 0f9cd962c..6367f19bc 100644 --- a/crates/storage/src/backend/tests.rs +++ b/crates/storage/src/backend/tests.rs @@ -14,6 +14,7 @@ use crate::api::{StorageBackend, Table}; /// Run the full test suite against a backend. pub fn run_backend_tests(backend: &dyn StorageBackend) { test_put_and_get(backend); + test_put_from_borrowed_bytes(backend); test_delete(backend); test_prefix_iterator(backend); test_nonexistent_key(backend); @@ -47,6 +48,23 @@ fn test_put_and_get(backend: &dyn StorageBackend) { } } +fn test_put_from_borrowed_bytes(backend: &dyn StorageBackend) { + let value = b"borrowed_value".to_vec(); + { + let mut batch = backend.begin_write().unwrap(); + batch + .put(Table::BlockHeaders, b"test_put_borrowed_key", &value) + .unwrap(); + batch.commit().unwrap(); + } + + let view = backend.begin_read().unwrap(); + let stored = view + .get(Table::BlockHeaders, b"test_put_borrowed_key") + .unwrap(); + assert_eq!(stored, Some(value), "the caller's buffer is still its own"); +} + fn test_contains(backend: &dyn StorageBackend) { { let mut batch = backend.begin_write().unwrap(); diff --git a/crates/storage/src/state_writer.rs b/crates/storage/src/state_writer.rs index 51e5f6b0e..e22719ae0 100644 --- a/crates/storage/src/state_writer.rs +++ b/crates/storage/src/state_writer.rs @@ -81,12 +81,16 @@ pub(crate) struct StateWrite { /// `Table::StateDiffs` value. `Some` for every lean state, and for a /// beacon state that is not an anchor. pub diff: Option>, - /// Beacon only: the target's [`encode_state_value`] bytes, for the - /// writer's snapshot memo. `Some` exactly when the write is a snapshot: - /// only a snapshot is ever a diff base, so a diff's own bytes are never - /// kept. `None` on the lean arm, which diffs in the field domain and never - /// reads a base's bytes. - pub encoded: Option>, + /// Beacon only: `snapshot` also becomes the writer's snapshot memo, the + /// base the next diffs are taken against. True exactly when a beacon state + /// is written as a snapshot: only a snapshot is ever a diff base, so a + /// diff's own bytes are never kept. False on the lean arm, which diffs in + /// the field domain and never reads a base's bytes. + /// + /// A flag rather than a second copy of the bytes: the writer puts the + /// snapshot into its batch from a borrow and then moves that same buffer + /// into the memo, so a mainnet state is never held twice for it. + pub memoize_snapshot: bool, } /// States handed to the writer but not yet committed to the backend. @@ -504,7 +508,7 @@ pub(crate) fn plan_lean_state_write(state: State, parent: &State) -> StateWrite StateWrite { snapshot, diff: Some(diff), - encoded: None, + memoize_snapshot: false, } } @@ -531,9 +535,9 @@ pub(crate) fn plan_beacon_state_write( let target = encode_state_value(state); match base { None => StateWrite { - snapshot: Some(target.clone()), + snapshot: Some(target), diff: None, - encoded: Some(target), + memoize_snapshot: true, }, Some((snapshot_root, base)) => { let delta = beacon_state_delta::encode(&target, base); @@ -552,7 +556,7 @@ pub(crate) fn plan_beacon_state_write( StateWrite { snapshot: None, diff: Some(framed), - encoded: None, + memoize_snapshot: false, } } } @@ -693,17 +697,29 @@ impl StateWriter { .put_batch(Table::StateDiffs, vec![(key.clone(), diff)]) .expect("put state diff"); } - if let Some(snapshot) = write.snapshot { - batch - .put_batch(Table::States, vec![(key, snapshot)]) - .expect("put state snapshot"); - } + // A snapshot the memo keeps is put from a borrow, so the same buffer + // can then move into the memo; any other value is handed over. + let memo = match write.snapshot { + Some(snapshot) if write.memoize_snapshot => { + batch + .put(Table::States, &key, &snapshot) + .expect("put state snapshot"); + Some(snapshot) + } + Some(snapshot) => { + batch + .put_batch(Table::States, vec![(key, snapshot)]) + .expect("put state snapshot"); + None + } + None => None, + }; batch.commit().expect("commit"); if matches!(&*request.state, BeaconState::Lean(_)) { return; } - match write.encoded { + match memo { // A snapshot is its own base, and the next diffs are taken against it. Some(encoded) => { self.snapshot_memo = Some((root, encoded)); @@ -962,7 +978,7 @@ mod tests { Some(expected.as_slice()), "the snapshot holds the anchored state's own bytes, not its parent's" ); - assert!(plan.encoded.is_none(), "the lean arm keeps no encoded memo"); + assert!(!plan.memoize_snapshot, "the lean arm keeps no encoded memo"); let mut inside = base_state(); inside.slot = interval; @@ -986,10 +1002,9 @@ mod tests { "no base means nothing to diff against" ); assert!(plan.diff.is_none()); - assert_eq!( - plan.encoded.as_deref(), - plan.snapshot.as_deref(), - "the memo is the very bytes that were snapshotted" + assert!( + plan.memoize_snapshot, + "the snapshotted bytes become the memo" ); } @@ -1014,7 +1029,7 @@ mod tests { "a base means a diff, not a snapshot" ); assert!( - plan.encoded.is_none(), + !plan.memoize_snapshot, "only a snapshot is a diff base, so a diff keeps no memo bytes" ); let framed = plan.diff.expect("a non-anchor records a diff"); From 3525a51586e46fc62177be87b601e70f77ef2cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:42:20 -0300 Subject: [PATCH 6/8] test(types): say why the no-regrow tests assert an exact capacity The automated review read the capacity equality as too strict and asked for >=. A regrow can only raise the capacity, so a lower bound would pass every regrow the tests exist to catch. The comments say so, and that the exactness is std Vec's behaviour rather than a documented promise of reserve_exact. --- .../state_transition/src/beacon/helpers/test_state.rs | 4 ++++ crates/storage/src/state_codec.rs | 3 +++ 2 files changed, 7 insertions(+) diff --git a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs index d939fa20d..00ba2e1d6 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs @@ -678,6 +678,10 @@ mod tests { let capacity = state.ssz_append_capacity(); let encoded = state.to_ssz(); + // Equality, not `>=`: a regrow can only raise the capacity, so a + // lower bound could never fail. `reserve_exact` promises only "at + // least" what it is asked for, but std's `Vec` takes exactly that, + // so any capacity above it here is a regrow. assert_eq!(encoded.capacity(), capacity, "{fork}: to_ssz regrew"); let decoded = BeaconState::from_ssz(fork, &encoded).expect("the encoding decodes"); assert_eq!(decoded, state, "{fork}"); diff --git a/crates/storage/src/state_codec.rs b/crates/storage/src/state_codec.rs index 8278f6326..5d6df8b22 100644 --- a/crates/storage/src/state_codec.rs +++ b/crates/storage/src/state_codec.rs @@ -76,6 +76,9 @@ mod tests { assert_eq!(bytes[0], ForkName::Electra.selector()); assert_eq!(bytes[1..], state.to_ssz()); + // Equality, for the reason state_transition's + // `ssz_append_never_regrows_a_buffer_with_its_capacity_to_spare` gives: + // a regrow only raises the capacity, so `>=` would pass one. assert_eq!( bytes.capacity(), 1 + state.ssz_append_capacity(), From eeff0fe2215459944e3179f9bc0108bc6aed86d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:24:39 -0300 Subject: [PATCH 7/8] test(types): detect a regrow by the buffer's address, not an exact capacity Three reviewers in the second automated round still read the exact-capacity assertions as fragile, comment or not: reserve_exact promises only at least what it is asked for. The no-regrow test now reserves the advertised headroom itself, with and without a prefix, and asserts the buffer's address and capacity are the same after the append, which holds whatever reserve_exact rounds to. Dropping the headroom from ssz_append_capacity fails it at phase0. The codec test keeps a lower bound only: its buffer holds ssz_append's headroom, so the encoding went in directly rather than being made apart and copied in. Whether the headroom suffices is the test above. --- .../src/beacon/helpers/test_state.rs | 30 ++++++++++++------- crates/storage/src/state_codec.rs | 16 +++++----- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs index 00ba2e1d6..f80c47742 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs @@ -678,20 +678,28 @@ mod tests { let capacity = state.ssz_append_capacity(); let encoded = state.to_ssz(); - // Equality, not `>=`: a regrow can only raise the capacity, so a - // lower bound could never fail. `reserve_exact` promises only "at - // least" what it is asked for, but std's `Vec` takes exactly that, - // so any capacity above it here is a regrow. - assert_eq!(encoded.capacity(), capacity, "{fork}: to_ssz regrew"); let decoded = BeaconState::from_ssz(fork, &encoded).expect("the encoding decodes"); assert_eq!(decoded, state, "{fork}"); - // Appending after a prefix reserves past the prefix, not from the - // start of the buffer. - let mut framed = vec![0xAB]; - state.ssz_append(&mut framed); - assert_eq!(framed.capacity(), 1 + capacity, "{fork}: ssz_append regrew"); - assert_eq!(framed[1..], encoded, "{fork}"); + // Given the headroom it advertises, the append must not reallocate: + // the buffer's address and capacity are the same before and after. + // That holds whatever `reserve_exact` rounds to, which its docs + // leave open, so the test does not lean on std's exact capacities. + // With no prefix this is `to_ssz`'s own path; with one, the + // storage layer's, where the encoding starts past a fork tag. + for prefix in [&[][..], &[0xAB][..]] { + let mut buf = prefix.to_vec(); + buf.reserve_exact(capacity); + let before = (buf.as_ptr(), buf.capacity()); + state.ssz_append(&mut buf); + let after = (buf.as_ptr(), buf.capacity()); + let len = prefix.len(); + assert_eq!( + after, before, + "{fork}: ssz_append regrew after {len} prefix bytes" + ); + assert_eq!(buf[len..], encoded, "{fork}"); + } } } } diff --git a/crates/storage/src/state_codec.rs b/crates/storage/src/state_codec.rs index 5d6df8b22..42e6f4df9 100644 --- a/crates/storage/src/state_codec.rs +++ b/crates/storage/src/state_codec.rs @@ -76,13 +76,15 @@ mod tests { assert_eq!(bytes[0], ForkName::Electra.selector()); assert_eq!(bytes[1..], state.to_ssz()); - // Equality, for the reason state_transition's - // `ssz_append_never_regrows_a_buffer_with_its_capacity_to_spare` gives: - // a regrow only raises the capacity, so `>=` would pass one. - assert_eq!( - bytes.capacity(), - 1 + state.ssz_append_capacity(), - "the buffer was regrown" + // The buffer holds `ssz_append`'s headroom, so the encoding went into + // it directly rather than being made apart and copied in (which would + // size it to the encoding alone). That the headroom suffices, so the + // append never regrows the buffer, is state_transition's + // `ssz_append_never_regrows_a_buffer_with_its_capacity_to_spare`. + let tag_and_headroom = 1 + state.ssz_append_capacity(); + assert!( + bytes.capacity() >= tag_and_headroom, + "the encoding was not appended in place" ); assert_eq!(decode_state_value(&bytes), state); } From cec0651f190e91dcd67ccdf622c49e65dfc9177e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:24:39 -0300 Subject: [PATCH 8/8] docs(storage): note the memo's spare capacity and assert its snapshot From the second automated review round: - StateWrite can express memoize_snapshot without a snapshot, which would leave the memo stale; the commit now debug-asserts it never happens. - memoize_snapshot says the memo keeps the encoder's headroom as spare capacity, and to_ssz that its capacity exceeds its length. - StorageWriteBatch::put says a backend that can copy from a borrow should override the cloning default. - largest_fixed_field says its lists are kept by hand and which test fails when a fork outgrows them. --- crates/common/types/src/beacon/containers/mod.rs | 7 ++++++- crates/storage/src/api/traits.rs | 4 +++- crates/storage/src/state_writer.rs | 10 +++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 0fb340705..8dd90e5f9 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -370,7 +370,8 @@ impl BeaconState { /// Goes through [`BeaconState::ssz_append`] rather than libssz's own /// `to_ssz`, which reserves only the encoding's length and so holds a fulu /// or gloas state twice while encoding it (see - /// [`BeaconState::ssz_append_capacity`]). + /// [`BeaconState::ssz_append_capacity`]). The returned buffer keeps that + /// headroom as spare capacity, so its `capacity()` exceeds its `len()`. pub fn to_ssz(&self) -> Vec { let mut buf = Vec::new(); self.ssz_append(&mut buf); @@ -405,6 +406,10 @@ impl BeaconState { /// a different container with fields a few hundred bytes wide, and /// reserving a beacon state's megabytes for it would often outweigh the /// state itself. + /// + /// The lists are kept by hand: a fork that adds a larger fixed-size field + /// must add it here. `ssz_append_never_regrows_a_buffer_with_its_capacity_to_spare` + /// (in `ethlambda-state-transition`, run on both presets) fails until it does. fn largest_fixed_field(&self) -> usize { use libssz::SszEncode; let largest = |sizes: &[usize]| sizes.iter().copied().max().unwrap_or(0); diff --git a/crates/storage/src/api/traits.rs b/crates/storage/src/api/traits.rs index e7f04643d..70332c2b3 100644 --- a/crates/storage/src/api/traits.rs +++ b/crates/storage/src/api/traits.rs @@ -102,7 +102,9 @@ pub trait StorageWriteBatch: Send { /// For a value the caller keeps after the write, so it need not clone one /// only to hand it over: a beacon snapshot is hundreds of megabytes, and /// RocksDB copies the value into its batch either way. The default goes - /// through [`put_batch`](Self::put_batch). + /// through [`put_batch`](Self::put_batch), cloning the value, so a backend + /// that can copy from a borrow should override it: on the snapshot path + /// the default brings back a full-state clone. fn put(&mut self, table: Table, key: &[u8], value: &[u8]) -> Result<(), Error> { self.put_batch(table, vec![(key.to_vec(), value.to_vec())]) } diff --git a/crates/storage/src/state_writer.rs b/crates/storage/src/state_writer.rs index e22719ae0..e39777549 100644 --- a/crates/storage/src/state_writer.rs +++ b/crates/storage/src/state_writer.rs @@ -89,7 +89,11 @@ pub(crate) struct StateWrite { /// /// A flag rather than a second copy of the bytes: the writer puts the /// snapshot into its batch from a borrow and then moves that same buffer - /// into the memo, so a mainnet state is never held twice for it. + /// into the memo, so a mainnet state is never held twice for it. The memo + /// therefore keeps that buffer's spare capacity, the encoder's headroom + /// (about 2 MiB on mainnet); `shrink_to_fit` would cost the copy this + /// saves. Never true without `snapshot`, which the commit asserts in debug + /// builds. pub memoize_snapshot: bool, } @@ -699,6 +703,10 @@ impl StateWriter { } // A snapshot the memo keeps is put from a borrow, so the same buffer // can then move into the memo; any other value is handed over. + debug_assert!( + !write.memoize_snapshot || write.snapshot.is_some(), + "a memoized snapshot needs a snapshot, or the memo is left stale" + ); let memo = match write.snapshot { Some(snapshot) if write.memoize_snapshot => { batch