From 7c5231e00c569cdbdbd103d9c215eee69da8e253 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:34:09 -0700 Subject: [PATCH 01/27] feat(common): add feature-gated rmp module scaffold --- crates/atuin-common/Cargo.toml | 3 +++ crates/atuin-common/src/lib.rs | 2 ++ crates/atuin-common/src/rmp.rs | 1 + 3 files changed, 6 insertions(+) create mode 100644 crates/atuin-common/src/rmp.rs diff --git a/crates/atuin-common/Cargo.toml b/crates/atuin-common/Cargo.toml index 6ae39b48761..2d01a11e864 100644 --- a/crates/atuin-common/Cargo.toml +++ b/crates/atuin-common/Cargo.toml @@ -18,6 +18,8 @@ test-utils = ["tracing", "dep:tracing-subscriber"] # The `string::ellipsis` module: budget-aware, grapheme/display-width-correct # string truncation. The module is compiled only when this feature is enabled. unicode = ["dep:unicode-width", "dep:unicode-segmentation"] +# The `rmp` module: MessagePack encode/decode extension traits and error types. +rmp = ["dep:rmp"] [dependencies] derive_more = { workspace = true } @@ -39,6 +41,7 @@ tracing = { workspace = true, optional = true } tracing-subscriber = { workspace = true, optional = true } unicode-width = { version = "0.2", optional = true } unicode-segmentation = { version = "1.11.0", optional = true } +rmp = { version = "0.8.14", optional = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/crates/atuin-common/src/lib.rs b/crates/atuin-common/src/lib.rs index c9e24658237..78eb4807a78 100644 --- a/crates/atuin-common/src/lib.rs +++ b/crates/atuin-common/src/lib.rs @@ -61,6 +61,8 @@ pub mod api; pub mod logs; pub mod path; pub mod record; +#[cfg(feature = "rmp")] +pub mod rmp; pub mod shell; pub mod string; #[cfg(feature = "test-utils")] diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs new file mode 100644 index 00000000000..861d710cb69 --- /dev/null +++ b/crates/atuin-common/src/rmp.rs @@ -0,0 +1 @@ +//! MessagePack encode/decode helpers built on [`rmp`]. From cee892d99f6e096b7eeb44e6c343db6a63e186ba Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:37:18 -0700 Subject: [PATCH 02/27] feat(common): add rmp EncodeError/DecodeError types --- crates/atuin-common/src/rmp.rs | 292 +++++++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs index 861d710cb69..bfb4ecbc16c 100644 --- a/crates/atuin-common/src/rmp.rs +++ b/crates/atuin-common/src/rmp.rs @@ -1 +1,293 @@ //! MessagePack encode/decode helpers built on [`rmp`]. +//! +//! `rmp`'s own error types are awkward: some don't implement [`Display`] for +//! every `E`, none say which variant they are, and the decode cursor offers no +//! owned-string read, no nil-aware optional read, and no structural checks +//! (array length, end-of-input). This module fills those gaps with two +//! extension traits — [`RmpDecodeExt`] for reading a [`Bytes`] cursor and +//! [`RmpEncodeExt`] for writing — plus [`DecodeError`] / [`EncodeError`], which +//! carry legible messages and convert cleanly into [`eyre::Report`]. +//! +//! Because every decode read returns [`DecodeError`], a decoder can use `?` +//! throughout and convert once at the boundary, rather than hand-rolling a +//! `map_err` at every read. +//! +//! [`Display`]: std::fmt::Display + +use rmp::Marker; +use rmp::decode::bytes::{Bytes, BytesReadError}; +use rmp::decode::{self, DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError}; +use rmp::encode::{self, RmpWrite, RmpWriteErr, ValueWriteError}; + +/// An error encountered while encoding a MessagePack value. +/// +/// Wraps [`ValueWriteError`] with a message that names the failing write and its +/// inner I/O error — neither of which `rmp`'s own [`Display`] reports. Implements +/// [`std::error::Error`], so it converts into [`eyre::Report`] with `?`. +/// +/// [`Display`]: std::fmt::Display +#[derive(Debug, derive_more::Display, derive_more::From, thiserror::Error)] +#[display("could not write MessagePack value: {_0:?}")] +pub struct EncodeError(ValueWriteError); + +/// An error encountered while decoding a MessagePack value. +/// +/// Wraps the three error types `rmp`'s decode functions return, and adds two +/// structural variants for checks `rmp` does not perform itself: +/// [`UnexpectedArrayLen`](Self::UnexpectedArrayLen) and +/// [`TrailingBytes`](Self::TrailingBytes). +/// +/// Converts into [`eyre::Report`] via a manual `From` rather than a +/// [`std::error::Error`] impl, because a [`DecodeError`] is not, in general, +/// `'static`. +/// +/// [`Display`]: std::fmt::Display +#[derive(Debug, derive_more::Display)] +pub enum DecodeError<'a, E: RmpReadErr = BytesReadError> { + /// The next value was not a valid UTF-8 string. + #[display("could not decode MessagePack string: {_0:?}")] + DecodeString(DecodeStringError<'a, E>), + /// The next value was not the expected number. + #[display("could not decode MessagePack number: {_0:?}")] + NumValueRead(NumValueReadError), + /// The next value could not be decoded. + #[display("could not decode MessagePack value: {_0:?}")] + ValueRead(ValueReadError), + /// An array marker did not have the expected length. + #[display("expected a MessagePack array of length {expected}, found {actual}")] + UnexpectedArrayLen { expected: u32, actual: u32 }, + /// Input remained after a value was fully decoded. + #[display("{remaining} trailing byte(s) after decoding MessagePack value")] + TrailingBytes { remaining: usize }, +} + +impl<'a, E: RmpReadErr> From> for DecodeError<'a, E> { + fn from(e: DecodeStringError<'a, E>) -> Self { + Self::DecodeString(e) + } +} + +impl From> for DecodeError<'_, E> { + fn from(e: NumValueReadError) -> Self { + Self::NumValueRead(e) + } +} + +impl From> for DecodeError<'_, E> { + fn from(e: ValueReadError) -> Self { + Self::ValueRead(e) + } +} + +impl DecodeError<'_, E> { + /// If this is a type mismatch, the [`Marker`] found instead of the expected + /// type. [`RmpDecodeExt::read_optional`] uses this to recognise nil. + pub fn type_mismatch(&self) -> Option { + match self { + Self::DecodeString(DecodeStringError::TypeMismatch(m)) + | Self::NumValueRead(NumValueReadError::TypeMismatch(m)) + | Self::ValueRead(ValueReadError::TypeMismatch(m)) => Some(*m), + _ => None, + } + } +} + +impl From> for eyre::Report { + fn from(e: DecodeError<'_, E>) -> Self { + eyre::eyre!("{e}") + } +} + +/// Reading helpers for a MessagePack [`Bytes`] cursor. +/// +/// Every method reports failures as [`DecodeError`], so decoders use `?` +/// throughout and convert once at the boundary. +pub trait RmpDecodeExt<'a> { + /// Run an `rmp` decode function, converting its error into [`DecodeError`]. + /// + /// Lets raw `rmp::decode` functions compose with `?`: + /// `bytes.read_with(rmp::decode::read_u64)?`. + fn read_with(&mut self, read: F) -> Result> + where + F: FnOnce(&mut Self) -> Result, + E: Into>; + + /// Read an owned [`String`]. + fn read_string(&mut self) -> Result>; + + /// Read a value that may be encoded as nil, returning [`None`] for nil. + /// + /// `read` decodes a `T`; if it fails specifically because it found + /// [`Marker::Null`], this yields [`None`] with the cursor left just past the + /// nil. Any other error is forwarded unchanged. + fn read_optional(&mut self, read: F) -> Result, DecodeError<'a>> + where + F: FnOnce(&mut Self) -> Result, + E: Into>; + + /// Read an array-length marker. + fn read_array_len(&mut self) -> Result>; + + /// Read an array-length marker and require it to equal `expected`. + /// + /// Returns [`DecodeError::UnexpectedArrayLen`] otherwise. For a + /// forward-compatible field count, use [`read_array_len`](Self::read_array_len) + /// and range-check the value yourself. + fn expect_array_len(&mut self, expected: u32) -> Result>; + + /// Succeed only if the cursor is at end-of-input, else + /// [`DecodeError::TrailingBytes`] — the standard malformed-record guard after + /// a fixed set of fields. + fn expect_eof(&self) -> Result<(), DecodeError<'a>>; +} + +impl<'a> RmpDecodeExt<'a> for Bytes<'a> { + fn read_with(&mut self, read: F) -> Result> + where + F: FnOnce(&mut Self) -> Result, + E: Into>, + { + read(self).map_err(Into::into) + } + + fn read_string(&mut self) -> Result> { + let slice = self.remaining_slice(); + let (string, rest) = match decode::read_str_from_slice(slice) { + Ok(pair) => pair, + Err(e) => { + if let DecodeStringError::TypeMismatch(_) = e { + // rmp's decode functions consume the marker byte on a type + // mismatch; do the same so `read_optional` can detect nil. + self.read_u8() + .expect("TypeMismatch implies the stream contains a marker byte"); + } + return Err(e.into()); + } + }; + *self = Bytes::new(rest); + Ok(string.into()) + } + + fn read_optional(&mut self, read: F) -> Result, DecodeError<'a>> + where + F: FnOnce(&mut Self) -> Result, + E: Into>, + { + match read(self) { + Ok(v) => Ok(Some(v)), + Err(e) => { + let e = e.into(); + if let Some(Marker::Null) = e.type_mismatch() { + Ok(None) + } else { + Err(e) + } + } + } + } + + fn read_array_len(&mut self) -> Result> { + decode::read_array_len(self).map_err(Into::into) + } + + fn expect_array_len(&mut self, expected: u32) -> Result> { + let actual = self.read_array_len()?; + if actual == expected { + Ok(actual) + } else { + Err(DecodeError::UnexpectedArrayLen { expected, actual }) + } + } + + fn expect_eof(&self) -> Result<(), DecodeError<'a>> { + let remaining = self.remaining_slice().len(); + if remaining == 0 { + Ok(()) + } else { + Err(DecodeError::TrailingBytes { remaining }) + } + } +} + +/// Writing helpers for a MessagePack output buffer. +pub trait RmpEncodeExt: RmpWrite { + /// Write an optional value, encoding [`None`] as [`Marker::Null`]. + /// + /// The mirror of [`RmpDecodeExt::read_optional`]: `write` encodes the inner + /// value when `value` is [`Some`]. + fn write_optional( + &mut self, + value: Option, + write: F, + ) -> Result<(), EncodeError> + where + F: FnOnce(&mut Self, T) -> Result<(), ValueWriteError>; +} + +impl RmpEncodeExt for W { + fn write_optional( + &mut self, + value: Option, + write: F, + ) -> Result<(), EncodeError> + where + F: FnOnce(&mut Self, T) -> Result<(), ValueWriteError>, + { + match value { + Some(v) => write(self, v).map_err(EncodeError::from), + None => encode::write_nil(self) + .map_err(|e| EncodeError::from(ValueWriteError::InvalidMarkerWrite(e))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use rmp::decode::bytes::Bytes; + use rstest::rstest; + + // Build a real DecodeError by asking rmp to read the wrong type. + fn type_mismatch_error<'a>() -> DecodeError<'a> { + // 0xc0 is the nil marker; reading it as a u64 is a type mismatch. + let mut bytes = Bytes::new(&[0xc0]); + rmp::decode::read_u64(&mut bytes) + .map_err(DecodeError::from) + .unwrap_err() + } + + #[test] + fn type_mismatch_exposes_marker() { + assert_eq!(type_mismatch_error().type_mismatch(), Some(rmp::Marker::Null)); + } + + #[rstest] + #[case::array_len( + DecodeError::<'static>::UnexpectedArrayLen { expected: 3, actual: 5 }, + None + )] + #[case::trailing(DecodeError::<'static>::TrailingBytes { remaining: 2 }, None)] + fn structural_variants_have_no_marker( + #[case] err: DecodeError<'static>, + #[case] expected: Option, + ) { + assert_eq!(err.type_mismatch(), expected); + } + + #[test] + fn decode_error_converts_to_eyre_with_message() { + let report: eyre::Report = + DecodeError::<'static, BytesReadError>::TrailingBytes { remaining: 4 }.into(); + assert!(report.to_string().contains("trailing")); + } + + #[test] + fn display_messages_are_legible() { + assert_eq!( + DecodeError::<'static, BytesReadError>::UnexpectedArrayLen { expected: 3, actual: 5 } + .to_string(), + "expected a MessagePack array of length 3, found 5", + ); + } +} From 6ffdd3a551b9fba56f59cd05cfafc095214b09aa Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:37:45 -0700 Subject: [PATCH 03/27] feat(common): add RmpDecodeExt read_with/read_string/read_optional --- crates/atuin-common/src/rmp.rs | 58 ++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs index bfb4ecbc16c..3038522bfd3 100644 --- a/crates/atuin-common/src/rmp.rs +++ b/crates/atuin-common/src/rmp.rs @@ -290,4 +290,62 @@ mod tests { "expected a MessagePack array of length 3, found 5", ); } + + // Encode helpers used only by tests, to build inputs. + fn enc)>(f: F) -> Vec { + let mut v = Vec::new(); + f(&mut v); + v + } + + #[test] + fn read_string_round_trips() { + let buf = enc(|v| rmp::encode::write_str(v, "héllo 🦀").unwrap()); + let mut bytes = Bytes::new(&buf); + assert_eq!(bytes.read_string().unwrap(), "héllo 🦀"); + assert!(bytes.remaining_slice().is_empty()); + } + + #[test] + fn read_string_on_wrong_type_errors_and_consumes_marker() { + // A lone nil marker: read_string must fail *and* consume the marker so a + // following read_optional can observe end-of-input correctly. + let mut bytes = Bytes::new(&[0xc0]); + assert!(bytes.read_string().is_err()); + assert!(bytes.remaining_slice().is_empty(), "marker byte must be consumed"); + } + + #[test] + fn read_with_converts_rmp_errors() { + let buf = enc(|v| rmp::encode::write_u64(v, 42).unwrap()); + let mut bytes = Bytes::new(&buf); + assert_eq!(bytes.read_with(rmp::decode::read_u64).unwrap(), 42); + } + + #[test] + fn read_optional_some_and_none() { + let some = enc(|v| rmp::encode::write_u64(v, 7).unwrap()); + let mut b = Bytes::new(&some); + assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), Some(7)); + + let none = enc(|v| rmp::encode::write_nil(v).unwrap()); + let mut b = Bytes::new(&none); + assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), None); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn read_optional_string_via_closure() { + let buf = enc(|v| rmp::encode::write_str(v, "x").unwrap()); + let mut b = Bytes::new(&buf); + assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), Some("x".to_string())); + } + + #[test] + fn read_optional_forwards_non_nil_errors() { + // A bool where a u64 is expected is a type mismatch that is NOT nil. + let buf = enc(|v| rmp::encode::write_bool(v, true).unwrap()); + let mut b = Bytes::new(&buf); + assert!(b.read_optional(rmp::decode::read_u64).is_err()); + } } From e86998a47b82c6d9975cb7211a43fa1e7f8bd664 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:38:58 -0700 Subject: [PATCH 04/27] feat(common): add RmpDecodeExt array-len and eof checks --- crates/atuin-common/src/rmp.rs | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs index 3038522bfd3..98ab463bdb2 100644 --- a/crates/atuin-common/src/rmp.rs +++ b/crates/atuin-common/src/rmp.rs @@ -348,4 +348,53 @@ mod tests { let mut b = Bytes::new(&buf); assert!(b.read_optional(rmp::decode::read_u64).is_err()); } + + fn array_of(len: u32) -> Vec { + enc(|v| { + rmp::encode::write_array_len(v, len).unwrap(); + }) + } + + #[test] + fn expect_array_len_exact_ok() { + let buf = array_of(3); + let mut b = Bytes::new(&buf); + assert_eq!(b.expect_array_len(3).unwrap(), 3); + } + + #[test] + fn expect_array_len_mismatch_reports_expected_and_actual() { + let buf = array_of(5); + let mut b = Bytes::new(&buf); + match b.expect_array_len(3) { + Err(DecodeError::UnexpectedArrayLen { expected, actual }) => { + assert_eq!((expected, actual), (3, 5)); + } + other => panic!("expected UnexpectedArrayLen, got {other:?}"), + } + } + + #[test] + fn read_array_len_returns_count_for_manual_range_checks() { + let buf = array_of(9); + let mut b = Bytes::new(&buf); + assert_eq!(b.read_array_len().unwrap(), 9); + } + + #[test] + fn expect_eof_ok_when_consumed() { + let buf = enc(|v| rmp::encode::write_u8(v, 1).unwrap()); + let mut b = Bytes::new(&buf); + b.read_with(rmp::decode::read_u8).unwrap(); + assert!(b.expect_eof().is_ok()); + } + + #[test] + fn expect_eof_reports_remaining() { + let b = Bytes::new(&[0x01, 0x02, 0x03]); + match b.expect_eof() { + Err(DecodeError::TrailingBytes { remaining }) => assert_eq!(remaining, 3), + other => panic!("expected TrailingBytes, got {other:?}"), + } + } } From c9ab60ae96545e0b9ae76f0ae4432424639abbf2 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:39:20 -0700 Subject: [PATCH 05/27] feat(common): add RmpEncodeExt write_optional --- crates/atuin-common/src/rmp.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs index 98ab463bdb2..2aa155ea73f 100644 --- a/crates/atuin-common/src/rmp.rs +++ b/crates/atuin-common/src/rmp.rs @@ -397,4 +397,28 @@ mod tests { other => panic!("expected TrailingBytes, got {other:?}"), } } + + #[test] + fn write_optional_some_then_read_back() { + let mut out = Vec::new(); + out.write_optional(Some(99u64), rmp::encode::write_u64).unwrap(); + let mut b = Bytes::new(&out); + assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), Some(99)); + } + + #[test] + fn write_optional_none_writes_nil() { + let mut out = Vec::new(); + out.write_optional::(None, rmp::encode::write_u64).unwrap(); + let mut b = Bytes::new(&out); + assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), None); + } + + #[test] + fn write_optional_str() { + let mut out = Vec::new(); + out.write_optional(Some("hi"), rmp::encode::write_str).unwrap(); + let mut b = Bytes::new(&out); + assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), Some("hi".to_string())); + } } From 9f6ee97326b5f7d3e3872a59e0d2ef9c960312d0 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:39:59 -0700 Subject: [PATCH 06/27] test(common): property tests for rmp round-trips and panic-safety --- crates/atuin-common/src/rmp.rs | 106 +++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 10 deletions(-) diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs index 2aa155ea73f..c1e484e3133 100644 --- a/crates/atuin-common/src/rmp.rs +++ b/crates/atuin-common/src/rmp.rs @@ -16,7 +16,9 @@ use rmp::Marker; use rmp::decode::bytes::{Bytes, BytesReadError}; -use rmp::decode::{self, DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError}; +use rmp::decode::{ + self, DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError, +}; use rmp::encode::{self, RmpWrite, RmpWriteErr, ValueWriteError}; /// An error encountered while encoding a MessagePack value. @@ -259,7 +261,10 @@ mod tests { #[test] fn type_mismatch_exposes_marker() { - assert_eq!(type_mismatch_error().type_mismatch(), Some(rmp::Marker::Null)); + assert_eq!( + type_mismatch_error().type_mismatch(), + Some(rmp::Marker::Null) + ); } #[rstest] @@ -285,8 +290,11 @@ mod tests { #[test] fn display_messages_are_legible() { assert_eq!( - DecodeError::<'static, BytesReadError>::UnexpectedArrayLen { expected: 3, actual: 5 } - .to_string(), + DecodeError::<'static, BytesReadError>::UnexpectedArrayLen { + expected: 3, + actual: 5 + } + .to_string(), "expected a MessagePack array of length 3, found 5", ); } @@ -312,7 +320,10 @@ mod tests { // following read_optional can observe end-of-input correctly. let mut bytes = Bytes::new(&[0xc0]); assert!(bytes.read_string().is_err()); - assert!(bytes.remaining_slice().is_empty(), "marker byte must be consumed"); + assert!( + bytes.remaining_slice().is_empty(), + "marker byte must be consumed" + ); } #[test] @@ -338,7 +349,10 @@ mod tests { fn read_optional_string_via_closure() { let buf = enc(|v| rmp::encode::write_str(v, "x").unwrap()); let mut b = Bytes::new(&buf); - assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), Some("x".to_string())); + assert_eq!( + b.read_optional(|b| b.read_string()).unwrap(), + Some("x".to_string()) + ); } #[test] @@ -401,7 +415,8 @@ mod tests { #[test] fn write_optional_some_then_read_back() { let mut out = Vec::new(); - out.write_optional(Some(99u64), rmp::encode::write_u64).unwrap(); + out.write_optional(Some(99u64), rmp::encode::write_u64) + .unwrap(); let mut b = Bytes::new(&out); assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), Some(99)); } @@ -409,7 +424,8 @@ mod tests { #[test] fn write_optional_none_writes_nil() { let mut out = Vec::new(); - out.write_optional::(None, rmp::encode::write_u64).unwrap(); + out.write_optional::(None, rmp::encode::write_u64) + .unwrap(); let mut b = Bytes::new(&out); assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), None); } @@ -417,8 +433,78 @@ mod tests { #[test] fn write_optional_str() { let mut out = Vec::new(); - out.write_optional(Some("hi"), rmp::encode::write_str).unwrap(); + out.write_optional(Some("hi"), rmp::encode::write_str) + .unwrap(); let mut b = Bytes::new(&out); - assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), Some("hi".to_string())); + assert_eq!( + b.read_optional(|b| b.read_string()).unwrap(), + Some("hi".to_string()) + ); + } + + use proptest::prelude::*; + + proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + // A string survives an encode -> read_string round trip exactly. + #[test] + fn string_round_trips(s in r"(?s).*") { + let buf = enc(|v| rmp::encode::write_str(v, &s).unwrap()); + let mut b = Bytes::new(&buf); + prop_assert_eq!(b.read_string().unwrap(), s); + prop_assert!(b.remaining_slice().is_empty()); + } + + // Option survives write_optional -> read_optional. + #[test] + fn optional_u64_round_trips(v in proptest::option::of(any::())) { + let mut out = Vec::new(); + out.write_optional(v, rmp::encode::write_u64).unwrap(); + let mut b = Bytes::new(&out); + prop_assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), v); + } + + // Option survives the closure-based optional round trip. + #[test] + fn optional_string_round_trips(v in proptest::option::of(r"(?s).*")) { + let mut out = Vec::new(); + out.write_optional(v.as_deref(), rmp::encode::write_str).unwrap(); + let mut b = Bytes::new(&out); + prop_assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), v); + } + + // A full array record (len + fields + optional tail) round trips, and the + // cursor is exactly exhausted afterwards. + #[test] + fn record_round_trips( + id in r"[a-z]{0,16}", + ts in any::(), + deleted in proptest::option::of(any::()), + ) { + let mut out = Vec::new(); + rmp::encode::write_array_len(&mut out, 3).unwrap(); + rmp::encode::write_str(&mut out, &id).unwrap(); + rmp::encode::write_u64(&mut out, ts).unwrap(); + out.write_optional(deleted, rmp::encode::write_u64).unwrap(); + + let mut b = Bytes::new(&out); + prop_assert_eq!(b.expect_array_len(3).unwrap(), 3); + prop_assert_eq!(b.read_string().unwrap(), id); + prop_assert_eq!(b.read_with(rmp::decode::read_u64).unwrap(), ts); + prop_assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), deleted); + prop_assert!(b.expect_eof().is_ok()); + } + + // Reads never panic on arbitrary bytes — they return Err instead. + #[test] + fn reads_never_panic(raw in proptest::collection::vec(any::(), 0..64)) { + let mut b = Bytes::new(&raw); + let _ = b.read_string(); + let mut b = Bytes::new(&raw); + let _ = b.read_array_len(); + let mut b = Bytes::new(&raw); + let _ = b.read_optional(rmp::decode::read_u64); + } } } From 27bc8b2e36ae307b64b0c4b8a1fc964921cc98df Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:40:18 -0700 Subject: [PATCH 07/27] chore(common): record rmp dependency in Cargo.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 6ebfd629d22..702daf1a015 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -399,6 +399,7 @@ dependencies = [ "getrandom 0.2.17", "pretty_assertions", "proptest", + "rmp", "rstest", "rustls", "semver", From c9364a339dbc514dd8e4d7c4fc3b20bf0558fb88 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:46:49 -0700 Subject: [PATCH 08/27] test(common): cover nil-optional cursor advance --- crates/atuin-common/src/rmp.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs index c1e484e3133..0d40efc4f3c 100644 --- a/crates/atuin-common/src/rmp.rs +++ b/crates/atuin-common/src/rmp.rs @@ -442,6 +442,21 @@ mod tests { ); } + #[test] + fn read_optional_nil_string_advances_to_next_field() { + // A nil optional-string field followed by a u64. The nil read must consume + // exactly the marker so the following field decodes correctly. + let mut out = Vec::new(); + out.write_optional::<&str, _>(None, rmp::encode::write_str) + .unwrap(); + rmp::encode::write_u64(&mut out, 1234).unwrap(); + + let mut b = Bytes::new(&out); + assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), None); + assert_eq!(b.read_with(rmp::decode::read_u64).unwrap(), 1234); + assert!(b.expect_eof().is_ok()); + } + use proptest::prelude::*; proptest! { @@ -472,6 +487,7 @@ mod tests { out.write_optional(v.as_deref(), rmp::encode::write_str).unwrap(); let mut b = Bytes::new(&out); prop_assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), v); + prop_assert!(b.remaining_slice().is_empty()); } // A full array record (len + fields + optional tail) round trips, and the From 70b935b58305f9aa6b85dc25451c3ba04e04542d Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:49:16 -0700 Subject: [PATCH 09/27] refactor(client): use atuin_common::rmp in history serde --- crates/atuin-client/Cargo.toml | 2 +- crates/atuin-client/src/history.rs | 37 +++++++++++++++--------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/crates/atuin-client/Cargo.toml b/crates/atuin-client/Cargo.toml index 19034870f4e..f246e073657 100644 --- a/crates/atuin-client/Cargo.toml +++ b/crates/atuin-client/Cargo.toml @@ -79,7 +79,7 @@ strum = { version = "0.27", features = ["strum_macros"] } [dependencies.atuin-common] path = "../atuin-common" version = "18.17.1" -features = ["tracing"] +features = ["tracing", "rmp"] [dev-dependencies] tokio = { version = "1", features = ["full"] } diff --git a/crates/atuin-client/src/history.rs b/crates/atuin-client/src/history.rs index 313a380f606..a0b5bae3e39 100644 --- a/crates/atuin-client/src/history.rs +++ b/crates/atuin-client/src/history.rs @@ -10,7 +10,7 @@ use eyre::{Result, bail}; use crate::secrets::SECRET_PATTERNS_RE; use crate::settings::Settings; use crate::utils::get_host_user; -use crate::utils::rmp::{DecodeError, EncodeError, read_optional, read_string, write_optional}; +use atuin_common::rmp::{EncodeError, RmpDecodeExt as _, RmpEncodeExt as _}; use time::OffsetDateTime; pub(crate) mod builder; @@ -230,14 +230,13 @@ impl History { encode::write_str(&mut output, &self.session)?; encode::write_str(&mut output, &self.hostname)?; - write_optional( - &mut output, + output.write_optional( self.deleted_at.map(|d| d.unix_timestamp_nanos() as u64), encode::write_u64, )?; encode::write_str(&mut output, self.author.as_str())?; - write_optional(&mut output, self.intent.as_deref(), encode::write_str)?; - write_optional(&mut output, self.shell.as_deref(), encode::write_str)?; + output.write_optional(self.intent.as_deref(), encode::write_str)?; + output.write_optional(self.shell.as_deref(), encode::write_str)?; Ok(DecryptedData(output)) } @@ -248,30 +247,30 @@ impl History { let mut bytes = Bytes::new(bytes); - let real_version = decode::read_u16(&mut bytes).map_err(DecodeError::from)?; + let real_version = bytes.read_with(decode::read_u16)?; if real_version != version.as_int() { bail!("expected to decode {version} record, found v{real_version}"); } - let nfields = decode::read_array_len(&mut bytes).map_err(DecodeError::from)?; + let nfields = bytes.read_array_len()?; let min_fields = version.min_fields(); if nfields < min_fields || version.max_fields().is_some_and(|max| nfields > max) { bail!("unexpected number of fields ({nfields}) for history version {version}"); } - let id = read_string(&mut bytes)?; - let timestamp = decode::read_u64(&mut bytes).map_err(DecodeError::from)?; - let duration = decode::read_int(&mut bytes).map_err(DecodeError::from)?; - let exit = decode::read_int(&mut bytes).map_err(DecodeError::from)?; + let id = bytes.read_string()?; + let timestamp = bytes.read_with(decode::read_u64)?; + let duration = bytes.read_with(decode::read_int)?; + let exit = bytes.read_with(decode::read_int)?; - let command = read_string(&mut bytes)?; - let cwd = read_string(&mut bytes)?; - let session = read_string(&mut bytes)?; - let hostname = read_string(&mut bytes)?; - let deleted_at = read_optional(&mut bytes, decode::read_u64)?; + let command = bytes.read_string()?; + let cwd = bytes.read_string()?; + let session = bytes.read_string()?; + let hostname = bytes.read_string()?; + let deleted_at = bytes.read_optional(decode::read_u64)?; let author = if version >= Version::One { - read_optional(&mut bytes, read_string)? + bytes.read_optional(|b| b.read_string())? } else { None }; @@ -281,13 +280,13 @@ impl History { Version::One => nfields > min_fields, Version::Two => true, } { - read_optional(&mut bytes, read_string)? + bytes.read_optional(|b| b.read_string())? } else { None }; let shell = if version >= Version::Two { - read_optional(&mut bytes, read_string)? + bytes.read_optional(|b| b.read_string())? } else { None }; From bef72f1777966bd76d597900d48aae790b90c5fd Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:50:15 -0700 Subject: [PATCH 10/27] refactor(client): delete now-unused utils/rmp module --- crates/atuin-client/src/utils.rs | 2 - crates/atuin-client/src/utils/rmp.rs | 123 -- docs/plans/2026-07-07-sync-bench-infra.md | 486 ++++++++ ...026-07-18-commonize-rmp-msgpack-helpers.md | 1014 +++++++++++++++++ 4 files changed, 1500 insertions(+), 125 deletions(-) delete mode 100644 crates/atuin-client/src/utils/rmp.rs create mode 100644 docs/plans/2026-07-07-sync-bench-infra.md create mode 100644 docs/plans/2026-07-18-commonize-rmp-msgpack-helpers.md diff --git a/crates/atuin-client/src/utils.rs b/crates/atuin-client/src/utils.rs index 236abc7e170..35d7db26724 100644 --- a/crates/atuin-client/src/utils.rs +++ b/crates/atuin-client/src/utils.rs @@ -1,5 +1,3 @@ -pub(crate) mod rmp; - pub(crate) fn get_hostname() -> String { std::env::var("ATUIN_HOST_NAME") .unwrap_or_else(|_| whoami::hostname().unwrap_or_else(|_| "unknown-host".to_string())) diff --git a/crates/atuin-client/src/utils/rmp.rs b/crates/atuin-client/src/utils/rmp.rs deleted file mode 100644 index 43b93d92efd..00000000000 --- a/crates/atuin-client/src/utils/rmp.rs +++ /dev/null @@ -1,123 +0,0 @@ -use rmp::Marker; -use rmp::decode::bytes::{Bytes, BytesReadError}; -use rmp::decode::{ - self, DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError, -}; -use rmp::encode::{self, RmpWrite, RmpWriteErr, ValueWriteError}; - -/// An error encountered while trying to encode a message with [`rmp`]. -/// -/// This is currently just a wrapper around [`ValueWriteError`] with a better error message. -/// [`rmp`]'s error message does not indicate which variant the error is (`InvalidMarkerWrite` or -/// `InvalidDataWrite`) and does not print anything about the inner I/O error of type `E`. -#[derive(Debug, derive_more::Display, derive_more::From, thiserror::Error)] -#[display("could not write MessagePack value: {_0:?}")] -pub struct EncodeError(ValueWriteError); - -/// An error encountered while trying to decode a message with [`rmp`]. -/// -/// This is a wrapper the various types of errors that can be returned by [`rmp`]'s decoding -/// functions. Unlike those types, this type implements [`Display`] with an error message that -/// indicates which variant the error is ([`rmp`]'s error types are enums; some unconditionally -/// print a static string and others don't even implement [`Display`] for all `E`). -/// -/// Conversion to [`eyre::Report`] is supported. This cannot be done by implementing -/// [`std::error::Error`] because this type is not, in general, `'static`, so a manual -/// implementation is provided. -#[derive(Debug, derive_more::Display, derive_more::From)] -#[display("could not decode MessagePack value: {_0:?}")] -pub enum DecodeError<'a, E: RmpReadErr = BytesReadError> { - DecodeString(DecodeStringError<'a, E>), - NumValueRead(NumValueReadError), - ValueRead(ValueReadError), -} - -impl DecodeError<'_, E> { - pub fn type_mismatch(&self) -> Option { - match self { - Self::DecodeString(DecodeStringError::TypeMismatch(m)) => Some(*m), - Self::NumValueRead(NumValueReadError::TypeMismatch(m)) => Some(*m), - Self::ValueRead(ValueReadError::TypeMismatch(m)) => Some(*m), - _ => None, - } - } -} - -impl From> for eyre::Report { - fn from(e: DecodeError<'_, E>) -> Self { - eyre::eyre!("{e}") - } -} - -/// Read an owned string from a [`Bytes`] object. -/// -/// If you need an owned [`String`], this function is more convenient than using -/// [`read_str_from_slice`] and converting the resulting [`str`], as you don't need to -/// keep unwrapping and re-creating the [`Bytes`] object. -/// -/// [`read_str_from_slice`]: decode::read_str_from_slice -pub fn read_string<'a>(bytes: &mut Bytes<'a>) -> Result> { - let slice = bytes.remaining_slice(); - let (string, rest) = match decode::read_str_from_slice(slice) { - Ok(pair) => pair, - Err(e) => { - if let DecodeStringError::TypeMismatch(_) = e { - // The decode functions in `rmp::decode` consume the marker byte when there's a - // type mismatch; make sure we do that too, as `read_optional` depends on it. - bytes - .read_u8() - .expect("TypeMismatch implies stream contains a marker byte"); - } - return Err(e.into()); - } - }; - *bytes = Bytes::new(rest); - Ok(string.into()) -} - -/// Read an optional value from the stream. -/// -/// This function calls `read`, which should try to decode a value of type `T` from the stream. If -/// that function returns an error indicating [`Marker::Null`] was encountered instead, this -/// function returns [`None`]. All other errors are forwarded as-is. -pub fn read_optional<'a, R, F, T, E>( - input: &mut R, - read: F, -) -> Result, DecodeError<'a, R::Error>> -where - R: RmpRead, - R::Error: Send + Sync, - F: FnOnce(&mut R) -> Result, - E: Into>, -{ - let err = match read(input) { - Ok(v) => return Ok(Some(v)), - Err(e) => e.into(), - }; - - if let Some(Marker::Null) = err.type_mismatch() { - Ok(None) - } else { - Err(err) - } -} - -/// Write an optional value to the stream. -/// -/// If `value` is [`Some`], this function calls `write` with the value, which should encode a value -/// of type `T` to the stream. Otherwise, this function writes [`Marker::Null`]. -pub fn write_optional( - output: &mut W, - value: Option, - write: F, -) -> Result<(), ValueWriteError> -where - W: RmpWrite, - F: FnOnce(&mut W, T) -> Result<(), ValueWriteError>, - W::Error: Send + Sync, -{ - match value { - Some(v) => write(output, v), - None => encode::write_nil(output).map_err(ValueWriteError::InvalidMarkerWrite), - } -} diff --git a/docs/plans/2026-07-07-sync-bench-infra.md b/docs/plans/2026-07-07-sync-bench-infra.md new file mode 100644 index 00000000000..966d9d3993f --- /dev/null +++ b/docs/plans/2026-07-07-sync-bench-infra.md @@ -0,0 +1,486 @@ +# Sync Benchmark Infrastructure Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Build benchmark infrastructure that exercises record sync over real HTTP, so page size changes (100→1000) show measurable round-trip savings. + +**Architecture:** The benchmark starts an in-process axum server backed by an in-memory `Database` implementation (no Postgres needed). A real `reqwest`-based `Client` talks to it over TCP loopback. The benchmark uploads N records, then measures the wall-clock cost of downloading them all at varying page sizes. This isolates HTTP round-trip overhead — the thing the page size change actually optimizes. + +The in-memory `Database` lives in a new crate `atuin-bench-support` to keep it reusable across bench targets. The benchmark itself lives in `crates/atuin/benches/` since the top-level `atuin` crate already has `atuin-server`, `atuin-server-database`, and `atuin-server-postgres` as dev-dependencies. + +**Tech Stack:** Rust, divan, axum (in-process), reqwest, tokio, `atuin-server`'s `launch_with_tcp_listener` + +--- + +### Task 1: Create `atuin-bench-support` crate with in-memory Database + +The `Database` trait (`atuin-server-database/src/lib.rs:114`) has ~20 methods. Only 8 are needed for record sync benchmarks. The rest return `DbError::NotFound` or empty results. + +**Files:** +- Create: `crates/atuin-bench-support/Cargo.toml` +- Create: `crates/atuin-bench-support/src/lib.rs` +- Modify: `Cargo.toml` (workspace members) + +**Step 1: Add crate to workspace** + +In the root `Cargo.toml`, add `"crates/atuin-bench-support"` to the `[workspace] members` list. + +**Step 2: Create `Cargo.toml`** + +```toml +[package] +name = "atuin-bench-support" +edition = "2024" +description = "In-memory server backend for benchmarking atuin sync" +publish = false + +rust-version = { workspace = true } +version = { workspace = true } + +[dependencies] +atuin-server-database = { workspace = true } +atuin-server = { workspace = true } +atuin-client = { path = "../atuin-client", version = "18.16.1", features = ["sync"] } +atuin-common = { workspace = true } +async-trait = { workspace = true } +tokio = { workspace = true, features = ["net", "sync", "time", "rt"] } +tracing = { workspace = true } +uuid = { workspace = true } +time = { workspace = true } +eyre = { workspace = true } +rand = { workspace = true } +``` + +**Step 3: Implement `InMemoryDb`** + +Create `crates/atuin-bench-support/src/lib.rs` with: + +```rust +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use async_trait::async_trait; +use atuin_common::record::{EncryptedData, HostId, Record, RecordIdx, RecordStatus}; +use atuin_server_database::models::{History, NewHistory, NewSession, NewUser, Session, User}; +use atuin_server_database::{calendar::TimePeriod, DbError, DbResult, DbSettings, Database}; +use time::{OffsetDateTime, UtcOffset}; + +#[derive(Default)] +struct Inner { + users: Vec, + sessions: Vec, + records: Vec>, + next_id: i64, +} + +#[derive(Clone, Default)] +pub struct InMemoryDb { + inner: Arc>, +} + +#[async_trait] +impl Database for InMemoryDb { + async fn new(_settings: &DbSettings) -> DbResult { + Ok(Self::default()) + } + + async fn get_session(&self, token: &str) -> DbResult { + let inner = self.inner.read().unwrap(); + inner + .sessions + .iter() + .find(|s| s.token == token) + .map(|s| Session { + id: s.id, + user_id: s.user_id, + token: s.token.clone(), + }) + .ok_or(DbError::NotFound) + } + + async fn get_session_user(&self, token: &str) -> DbResult { + let session = self.get_session(token).await?; + let inner = self.inner.read().unwrap(); + inner + .users + .iter() + .find(|u| u.id == session.user_id) + .map(|u| User { + id: u.id, + username: u.username.clone(), + email: u.email.clone(), + password: u.password.clone(), + }) + .ok_or(DbError::NotFound) + } + + async fn add_session(&self, session: &NewSession) -> DbResult<()> { + let mut inner = self.inner.write().unwrap(); + let id = inner.next_id; + inner.next_id += 1; + inner.sessions.push(Session { + id, + user_id: session.user_id, + token: session.token.clone(), + }); + Ok(()) + } + + async fn get_user(&self, username: &str) -> DbResult { + let inner = self.inner.read().unwrap(); + inner + .users + .iter() + .find(|u| u.username == username) + .map(|u| User { + id: u.id, + username: u.username.clone(), + email: u.email.clone(), + password: u.password.clone(), + }) + .ok_or(DbError::NotFound) + } + + async fn get_user_session(&self, u: &User) -> DbResult { + let inner = self.inner.read().unwrap(); + inner + .sessions + .iter() + .find(|s| s.user_id == u.id) + .map(|s| Session { + id: s.id, + user_id: s.user_id, + token: s.token.clone(), + }) + .ok_or(DbError::NotFound) + } + + async fn add_user(&self, user: &NewUser) -> DbResult { + let mut inner = self.inner.write().unwrap(); + let id = inner.next_id; + inner.next_id += 1; + inner.users.push(User { + id, + username: user.username.clone(), + email: user.email.clone(), + password: user.password.clone(), + }); + Ok(id) + } + + async fn update_user_password(&self, _u: &User) -> DbResult<()> { + Err(DbError::NotFound) + } + + async fn add_records(&self, _user: &User, records: &[Record]) -> DbResult<()> { + let mut inner = self.inner.write().unwrap(); + inner.records.extend(records.iter().cloned()); + Ok(()) + } + + async fn next_records( + &self, + _user: &User, + host: HostId, + tag: String, + start: Option, + count: u64, + ) -> DbResult>> { + let inner = self.inner.read().unwrap(); + let start = start.unwrap_or(0); + let results: Vec<_> = inner + .records + .iter() + .filter(|r| r.host == host && r.tag == tag && r.idx >= start) + .take(count as usize) + .cloned() + .collect(); + Ok(results) + } + + async fn status(&self, _user: &User) -> DbResult { + let inner = self.inner.read().unwrap(); + let mut status = RecordStatus::new(); + for record in &inner.records { + let current = status.hosts.entry(record.host).or_default(); + let tag_idx = current.entry(record.tag.clone()).or_insert(0); + if record.idx >= *tag_idx { + *tag_idx = record.idx + 1; + } + } + Ok(status) + } + + // --- Stubs for methods not needed by record sync --- + + async fn count_history(&self, _user: &User) -> DbResult { + Ok(0) + } + async fn count_history_cached(&self, _user: &User) -> DbResult { + Ok(0) + } + async fn delete_user(&self, _u: &User) -> DbResult<()> { + Err(DbError::NotFound) + } + async fn delete_history(&self, _user: &User, _id: String) -> DbResult<()> { + Err(DbError::NotFound) + } + async fn deleted_history(&self, _user: &User) -> DbResult> { + Ok(vec![]) + } + async fn delete_store(&self, _user: &User) -> DbResult<()> { + Ok(()) + } + async fn count_history_range( + &self, + _user: &User, + _range: std::ops::Range, + ) -> DbResult { + Ok(0) + } + async fn list_history( + &self, + _user: &User, + _created_after: OffsetDateTime, + _since: OffsetDateTime, + _host: &str, + _page_size: i64, + ) -> DbResult> { + Ok(vec![]) + } + async fn add_history(&self, _history: &[NewHistory]) -> DbResult<()> { + Ok(()) + } + async fn oldest_history(&self, _user: &User) -> DbResult { + Err(DbError::NotFound) + } +} +``` + +**Step 4: Verify it compiles** + +Run: `cargo check -p atuin-bench-support` +Expected: Clean compilation. + +**Step 5: Commit** + +```bash +git add crates/atuin-bench-support/ Cargo.toml +git commit -m "bench: add in-memory Database impl for sync benchmarks + +Implements the atuin-server-database Database trait backed by +Arc> storage. No Postgres needed — exercises real +HTTP round-trips through axum without external dependencies." +``` + +--- + +### Task 2: Add bench server helper to `atuin-bench-support` + +Add a `pub async fn start_bench_server()` that starts an in-process axum server with `InMemoryDb`, registers a test user, and returns a `Client` ready for benchmarking. + +**Files:** +- Modify: `crates/atuin-bench-support/src/lib.rs` + +**Step 1: Add the helper function** + +Append to `crates/atuin-bench-support/src/lib.rs`: + +```rust +use atuin_client::api_client::{self, AuthToken, Client}; +use atuin_server::{Settings as ServerSettings, launch_with_tcp_listener}; +use tokio::net::TcpListener; +use tokio::sync::oneshot; + +pub struct BenchServer { + pub addr: String, + _shutdown: oneshot::Sender<()>, + _handle: tokio::task::JoinHandle<()>, +} + +impl BenchServer { + pub async fn start() -> Self { + let settings = ServerSettings { + host: "127.0.0.1".to_owned(), + port: 0, + path: String::new(), + sync_v1_enabled: false, + open_registration: true, + max_history_length: 8192, + max_record_size: 1024 * 1024, + page_size: 1100, + register_webhook_url: None, + register_webhook_username: String::new(), + db_settings: DbSettings { + db_uri: "memory".to_owned(), + read_db_uri: None, + }, + metrics: atuin_server::settings::Metrics::default(), + fake_version: None, + }; + + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = format!("http://{}", listener.local_addr().unwrap()); + + let handle = tokio::spawn(async move { + launch_with_tcp_listener::( + settings, + listener, + shutdown_rx.unwrap_or_else(|_| ()), + ) + .await + .unwrap(); + }); + + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + Self { + addr, + _shutdown: shutdown_tx, + _handle: handle, + } + } + + pub async fn register_client(&self) -> Client<'_> { + let username = uuid::Uuid::new_v4().to_string(); + let email = format!("{username}@bench.test"); + let password = "bench-password"; + + let resp = api_client::register(&self.addr, &username, &email, password) + .await + .unwrap(); + + Client::new(&self.addr, AuthToken::Token(resp.session), 5, 30).unwrap() + } +} +``` + +**Step 2: Verify it compiles** + +Run: `cargo check -p atuin-bench-support` +Expected: Clean compilation. + +**Step 3: Commit** + +```bash +git add crates/atuin-bench-support/src/lib.rs +git commit -m "bench: add BenchServer helper for in-process sync benchmarks + +Starts an axum server on a random port with InMemoryDb, registers +a test user, and returns a ready-to-use Client. No external +services required." +``` + +--- + +### Task 3: Write the sync download benchmark + +The benchmark measures what the page size change actually optimizes: the wall-clock cost of downloading N records through HTTP at different page sizes. With page_size=100, downloading 1000 records takes 10 HTTP round-trips. With page_size=1000, it takes 1. + +**Files:** +- Create: `crates/atuin/benches/sync.rs` +- Modify: `crates/atuin/Cargo.toml` (add divan + bench target) + +**Step 1: Add dev-dependencies and bench target to `crates/atuin/Cargo.toml`** + +Add to `[dev-dependencies]`: +```toml +divan = "0.1.14" +atuin-bench-support = { path = "../atuin-bench-support" } +``` + +Add at end of file: +```toml +[[bench]] +name = "sync" +harness = false +``` + +**Step 2: Write the benchmark** + +Create `crates/atuin/benches/sync.rs`: + +```rust +use atuin_bench_support::BenchServer; +use atuin_common::record::{EncryptedData, HostId, Record}; +use atuin_common::utils::uuid_v7; + +fn main() { + divan::main(); +} + +fn generate_records(n: usize) -> (HostId, String, Vec>) { + let host = HostId(uuid_v7()); + let tag = "bench".to_string(); + + let records: Vec<_> = (0..n) + .map(|i| { + Record::builder() + .host(host) + .version("v0".to_string()) + .tag(tag.clone()) + .idx(i as u64) + .data(EncryptedData { + data: format!("bench-data-{i}"), + content_encryption_key: "bench-key".to_string(), + }) + .build() + }) + .collect(); + + (host, tag, records) +} + +/// Downloads 1000 pre-uploaded records at varying page sizes. +/// +/// This directly measures the HTTP round-trip overhead that the page size +/// constant controls. At page_size=100, 1000 records require 10 HTTP requests. +/// At page_size=1000, they require 1. +#[divan::bench(args = [100, 1000], min_time = 5, sample_count = 50)] +fn download_records(bencher: divan::Bencher, page_size: u64) { + let rt = tokio::runtime::Runtime::new().unwrap(); + + let server = rt.block_on(BenchServer::start()); + let client = rt.block_on(server.register_client()); + let (host, tag, records) = generate_records(1000); + rt.block_on(client.post_records(&records)).unwrap(); + + bencher.bench(|| { + rt.block_on(async { + let mut offset = 0u64; + loop { + let page = client + .next_records(host, tag.clone(), offset, page_size) + .await + .unwrap(); + if page.is_empty() { + break; + } + offset += page.len() as u64; + } + }) + }); +} +``` + +**Step 3: Verify it compiles** + +Run: `cargo bench -p atuin --bench sync -- --test` +Expected: Compiles and lists `download_records` with both args. + +**Step 4: Run benchmark** + +Run: `cargo bench -p atuin --bench sync` +Expected: Output showing `download_records/100` significantly slower than `download_records/1000` due to 10× more HTTP round-trips. + +**Step 5: Commit** + +```bash +git add crates/atuin/benches/sync.rs crates/atuin/Cargo.toml +git commit -m "bench: add sync download benchmark comparing page sizes + +Exercises real HTTP round-trips through an in-process axum server +with InMemoryDb. Downloads 1000 records at page_size 100 vs 1000, +directly measuring the round-trip overhead the page size controls." +``` diff --git a/docs/plans/2026-07-18-commonize-rmp-msgpack-helpers.md b/docs/plans/2026-07-18-commonize-rmp-msgpack-helpers.md new file mode 100644 index 00000000000..37c585349b4 --- /dev/null +++ b/docs/plans/2026-07-18-commonize-rmp-msgpack-helpers.md @@ -0,0 +1,1014 @@ +# Commonize MessagePack (`rmp`) Helpers into `atuin-common` Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Replace `atuin-client`'s private `utils/rmp.rs` with a rewritten, extension-trait-based `atuin_common::rmp` module (feature-gated behind `rmp`), give it rich rstest/proptest coverage and good docs, and migrate every caller onto it — deleting six copies of hand-rolled `error_report`/trailing-bytes/array-length boilerplate. + +**Architecture:** Two extension traits — `RmpDecodeExt<'a>` implemented for `rmp::decode::Bytes<'a>` (the one decode cursor used everywhere) and `RmpEncodeExt` blanket-implemented for every `RmpWrite` — plus two error types `DecodeError`/`EncodeError` that carry legible `Display` messages and convert into `eyre::Report`. All decode reads return `DecodeError` so callers use `?` throughout instead of `.map_err(error_report)` at every read. The module lives in `atuin-common` (a leaf crate) so `atuin-client`, `atuin-scripts`, `atuin-dotfiles`, and `atuin-kv` can all share it with no dependency cycle. + +**Tech Stack:** Rust 2024, `rmp = 0.8.14`, `derive_more = 2` (`display`, `from`, `error`), `thiserror`, `eyre`; tests use `rstest`, `proptest`, `pretty_assertions` (already `atuin-common` dev-deps). + +--- + +## Background & context (read before starting) + +**What exists today.** `crates/atuin-client/src/utils/rmp.rs` (123 lines, declared `pub(crate) mod rmp;` in `utils.rs:1`) provides `EncodeError`, `DecodeError`, `read_string`, `read_optional`, `write_optional`. Its **only** real consumer is `crates/atuin-client/src/history.rs`. + +**The duplication we are killing.** Six other files never import that module — they call raw `rmp` and each re-declare an identical local helper: +```rust +fn error_report(err: E) -> eyre::Report { eyre!("{err:?}") } +``` +Copies live in: `atuin-scripts/src/store/record.rs:52`, `atuin-client/src/history/store.rs:76`, `atuin-dotfiles/src/shell.rs:45`, `atuin-dotfiles/src/store.rs:58`, `atuin-dotfiles/src/store/var.rs:52`, `atuin-kv/src/store/record.rs:39`. (`encryption.rs` inlines the same as `.map_err(|err| eyre!("{err:?}"))`.) Alongside it, the same two idioms recur ~7× each: +- **trailing-bytes guard:** `if !bytes.is_empty() { bail!("trailing bytes ... malformed") }` +- **array-length assertion:** `let n = read_array_len(..)?; ensure!(n == N, "too many entries ...")` + +`DecodeError` already converts to `eyre::Report`, so the shared fix for `error_report` is simply "make decode reads return `DecodeError` and use `?`". `expect_array_len`/`expect_eof` absorb the other two idioms. + +**Concrete usage the interface must serve** (verified against source): +- Decode cursor is `rmp::decode::Bytes<'a>` in every file. (One kv site at `atuin-kv/src/store/record.rs:78` feeds a raw `&mut &[u8]` to `read_bool`; we normalize it to `Bytes` during migration.) +- Encode writer is `&mut Vec` everywhere. `Vec: RmpWrite` (proven: `history.rs:217` returns `Result<_, EncodeError>` — default `EncodeError` — while `?`-converting `write_optional`'s error). +- `history.rs:256-260` needs a **range** field-count check (version-dependent min/max), not exact — so it keeps its own logic via `read_array_len()`; `expect_array_len` is exact-only by design (YAGNI). +- `history.rs:274,284,290` passes `read_string` as a bare fn into `read_optional`. As a method, this becomes the closure `|b| b.read_string()`. +- `write_optional` is called with `encode::write_u64` and `encode::write_str` as the writer fn (`history.rs:233-240`). + +**Naming.** Match the existing high-quality modules (`EllipsizeExt`, `EscapeNonPrintablePosixExt`, `UrlAppendExt`): traits are `RmpDecodeExt` / `RmpEncodeExt`, `...Ext` suffix, blanket/receiver impls, doc-comment-per-public-item but no filler docs. + +**Module-vs-crate name.** Inside `atuin-common`, the module is `crate::rmp` and the crate is `rmp`. A leading `rmp::` path resolves to the extern crate (extern prelude), so `use rmp::decode` inside the module file is unambiguous. Consumers must `use atuin_common::rmp::RmpDecodeExt;` (import the *items*, never `use atuin_common::rmp;` as a whole, which would shadow the crate). + +**Staging.** Stage A (Tasks 1–8) rewrites + moves the module and migrates its one real consumer (`history.rs`), then deletes the old file — this alone satisfies "completely rewrite and move it," and the tree is green at the end of Task 8. Stage B (Tasks 9–13) is the commonization payoff: migrate the six raw-`rmp` files off their local `error_report`/idioms. Stage B is independently valuable and can be deferred, but it is why the module is worth commonizing. + +**Reference — full target module.** The complete intended contents of `crates/atuin-common/src/rmp.rs` (module code, not tests) are in **Appendix A** at the end of this document. Tasks 2–5 build it up test-first; use Appendix A as the definitive source for signatures, bounds, and doc comments. + +--- + +## Stage A — rewrite, move, and land the module + +### Task 1: Add the `rmp` feature, dependency, and empty module + +**Files:** +- Modify: `crates/atuin-common/Cargo.toml` +- Modify: `crates/atuin-common/src/lib.rs:60-63` (the `pub mod ...;` block) +- Create: `crates/atuin-common/src/rmp.rs` + +**Step 1: Add the feature and optional dependency** + +In `crates/atuin-common/Cargo.toml`, under `[features]` add (after the existing `unicode = [...]` line): +```toml +# The `rmp` module: MessagePack encode/decode extension traits and error types. +rmp = ["dep:rmp"] +``` +Under `[dependencies]` add (matching the `0.8.14` pin used by the other crates — there is no workspace `rmp` entry): +```toml +rmp = { version = "0.8.14", optional = true } +``` + +**Step 2: Create a stub module** + +`crates/atuin-common/src/rmp.rs`: +```rust +//! MessagePack encode/decode helpers built on [`rmp`]. +``` + +**Step 3: Wire it into the crate root** + +In `crates/atuin-common/src/lib.rs`, add alongside the other `pub mod` declarations: +```rust +#[cfg(feature = "rmp")] +pub mod rmp; +``` + +**Step 4: Verify it builds both ways** + +Run: `cargo build -p atuin-common --features rmp` +Expected: PASS. +Run: `cargo build -p atuin-common` +Expected: PASS (module absent, `rmp` crate not compiled). + +**Step 5: Commit** +```bash +git add crates/atuin-common/Cargo.toml crates/atuin-common/src/lib.rs crates/atuin-common/src/rmp.rs +git commit -m "feat(common): add feature-gated rmp module scaffold" +``` + +--- + +### Task 2: Port the error types (`EncodeError`, `DecodeError`) + +**Files:** +- Modify: `crates/atuin-common/src/rmp.rs` + +**Step 1: Write failing tests** + +Append to `crates/atuin-common/src/rmp.rs`: +```rust +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use rmp::decode::bytes::Bytes; + use rstest::rstest; + + // Build a real DecodeError by asking rmp to read the wrong type. + fn type_mismatch_error<'a>() -> DecodeError<'a> { + // 0xc0 is the nil marker; reading it as a u64 is a type mismatch. + let mut bytes = Bytes::new(&[0xc0]); + rmp::decode::read_u64(&mut bytes) + .map_err(DecodeError::from) + .unwrap_err() + } + + #[test] + fn type_mismatch_exposes_marker() { + assert_eq!(type_mismatch_error().type_mismatch(), Some(rmp::Marker::Null)); + } + + #[rstest] + #[case::array_len( + DecodeError::<'static>::UnexpectedArrayLen { expected: 3, actual: 5 }, + None + )] + #[case::trailing(DecodeError::<'static>::TrailingBytes { remaining: 2 }, None)] + fn structural_variants_have_no_marker( + #[case] err: DecodeError<'static>, + #[case] expected: Option, + ) { + assert_eq!(err.type_mismatch(), expected); + } + + #[test] + fn decode_error_converts_to_eyre_with_message() { + let report: eyre::Report = + DecodeError::<'static>::TrailingBytes { remaining: 4 }.into(); + assert!(report.to_string().contains("trailing")); + } + + #[test] + fn display_messages_are_legible() { + assert_eq!( + DecodeError::<'static>::UnexpectedArrayLen { expected: 3, actual: 5 }.to_string(), + "expected a MessagePack array of length 3, found 5", + ); + } +} +``` + +**Step 2: Run to verify failure** + +Run: `cargo test -p atuin-common --features rmp rmp::tests` +Expected: FAIL to compile (`DecodeError`/`EncodeError` not defined). + +**Step 3: Add the imports and error types** + +Insert the imports block and both error types from **Appendix A** (everything from the `use` lines through the `impl ... From for eyre::Report` block) above the `#[cfg(test)]` module. + +**Step 4: Run to verify pass** + +Run: `cargo test -p atuin-common --features rmp rmp::tests` +Expected: PASS. +Run: `cargo clippy -p atuin-common --features rmp -- -D warnings` +Expected: PASS. + +**Step 5: Commit** +```bash +git add crates/atuin-common/src/rmp.rs +git commit -m "feat(common): add rmp EncodeError/DecodeError types" +``` + +--- + +### Task 3: `RmpDecodeExt` core reads — `read_with`, `read_string`, `read_optional` + +**Files:** +- Modify: `crates/atuin-common/src/rmp.rs` + +**Step 1: Write failing tests** + +Add to the `tests` module: +```rust + // Encode helpers used only by tests, to build inputs. + fn enc)>(f: F) -> Vec { + let mut v = Vec::new(); + f(&mut v); + v + } + + #[test] + fn read_string_round_trips() { + let buf = enc(|v| rmp::encode::write_str(v, "héllo 🦀").unwrap()); + let mut bytes = Bytes::new(&buf); + assert_eq!(bytes.read_string().unwrap(), "héllo 🦀"); + assert!(bytes.remaining_slice().is_empty()); + } + + #[test] + fn read_string_on_wrong_type_errors_and_consumes_marker() { + // A lone nil marker: read_string must fail *and* consume the marker so a + // following read_optional can observe end-of-input correctly. + let mut bytes = Bytes::new(&[0xc0]); + assert!(bytes.read_string().is_err()); + assert!(bytes.remaining_slice().is_empty(), "marker byte must be consumed"); + } + + #[test] + fn read_with_converts_rmp_errors() { + let buf = enc(|v| rmp::encode::write_u64(v, 42).unwrap()); + let mut bytes = Bytes::new(&buf); + assert_eq!(bytes.read_with(rmp::decode::read_u64).unwrap(), 42); + } + + #[test] + fn read_optional_some_and_none() { + let some = enc(|v| rmp::encode::write_u64(v, 7).unwrap()); + let mut b = Bytes::new(&some); + assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), Some(7)); + + let none = enc(|v| rmp::encode::write_nil(v).unwrap()); + let mut b = Bytes::new(&none); + assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), None); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn read_optional_string_via_closure() { + let buf = enc(|v| rmp::encode::write_str(v, "x").unwrap()); + let mut b = Bytes::new(&buf); + assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), Some("x".to_string())); + } + + #[test] + fn read_optional_forwards_non_nil_errors() { + // A bool where a u64 is expected is a type mismatch that is NOT nil. + let buf = enc(|v| rmp::encode::write_bool(v, true).unwrap()); + let mut b = Bytes::new(&buf); + assert!(b.read_optional(rmp::decode::read_u64).is_err()); + } +``` + +**Step 2: Run to verify failure** + +Run: `cargo test -p atuin-common --features rmp rmp::tests` +Expected: FAIL (`RmpDecodeExt` / methods not defined). + +**Step 3: Implement the trait + these three methods** + +Add the `RmpDecodeExt<'a>` trait declaration and the `impl<'a> RmpDecodeExt<'a> for Bytes<'a>` block from **Appendix A**, but for this task include **only** `read_with`, `read_string`, and `read_optional` (leave `read_array_len`/`expect_array_len`/`expect_eof` for Task 4). Keep the trait declaration complete (all six method signatures) and add a `todo!()`-free partial impl — i.e. add the three later methods' signatures to the trait now but implement the remaining three in Task 4. To keep the crate compiling between tasks, add all six method bodies now using Appendix A (implementing all six here is fine; Task 4 just adds their tests). **Recommended:** implement all six method bodies in this task, and add the validation *tests* in Task 4. + +**Step 4: Run to verify pass** + +Run: `cargo test -p atuin-common --features rmp rmp::tests` +Expected: PASS. +Run: `cargo clippy -p atuin-common --features rmp -- -D warnings` +Expected: PASS. + +**Step 5: Commit** +```bash +git add crates/atuin-common/src/rmp.rs +git commit -m "feat(common): add RmpDecodeExt read_with/read_string/read_optional" +``` + +--- + +### Task 4: `RmpDecodeExt` validation — `read_array_len`, `expect_array_len`, `expect_eof` + +**Files:** +- Modify: `crates/atuin-common/src/rmp.rs` + +**Step 1: Write failing tests** + +Add to the `tests` module: +```rust + fn array_of(len: u32) -> Vec { + enc(|v| rmp::encode::write_array_len(v, len).unwrap()) + } + + #[test] + fn expect_array_len_exact_ok() { + let buf = array_of(3); + let mut b = Bytes::new(&buf); + assert_eq!(b.expect_array_len(3).unwrap(), 3); + } + + #[test] + fn expect_array_len_mismatch_reports_expected_and_actual() { + let buf = array_of(5); + let mut b = Bytes::new(&buf); + match b.expect_array_len(3) { + Err(DecodeError::UnexpectedArrayLen { expected, actual }) => { + assert_eq!((expected, actual), (3, 5)); + } + other => panic!("expected UnexpectedArrayLen, got {other:?}"), + } + } + + #[test] + fn read_array_len_returns_count_for_manual_range_checks() { + let buf = array_of(9); + let mut b = Bytes::new(&buf); + assert_eq!(b.read_array_len().unwrap(), 9); + } + + #[test] + fn expect_eof_ok_when_consumed() { + let buf = enc(|v| rmp::encode::write_u8(v, 1).unwrap()); + let mut b = Bytes::new(&buf); + b.read_with(rmp::decode::read_u8).unwrap(); + assert!(b.expect_eof().is_ok()); + } + + #[test] + fn expect_eof_reports_remaining() { + let b = Bytes::new(&[0x01, 0x02, 0x03]); + match b.expect_eof() { + Err(DecodeError::TrailingBytes { remaining }) => assert_eq!(remaining, 3), + other => panic!("expected TrailingBytes, got {other:?}"), + } + } +``` + +**Step 2: Run to verify failure** + +Run: `cargo test -p atuin-common --features rmp rmp::tests` +Expected: PASS if Task 3 already implemented all six bodies (recommended path); otherwise FAIL until you add the three method bodies from Appendix A. If FAIL, add them now. + +**Step 3: Ensure the three method bodies are present** + +Confirm `read_array_len`, `expect_array_len`, `expect_eof` match Appendix A. + +**Step 4: Run to verify pass** + +Run: `cargo test -p atuin-common --features rmp rmp::tests` +Expected: PASS. +Run: `cargo clippy -p atuin-common --features rmp -- -D warnings` +Expected: PASS. + +**Step 5: Commit** +```bash +git add crates/atuin-common/src/rmp.rs +git commit -m "feat(common): add RmpDecodeExt array-len and eof checks" +``` + +--- + +### Task 5: `RmpEncodeExt::write_optional` + +**Files:** +- Modify: `crates/atuin-common/src/rmp.rs` + +**Step 1: Write failing tests** + +Add to the `tests` module: +```rust + #[test] + fn write_optional_some_then_read_back() { + let mut out = Vec::new(); + out.write_optional(Some(99u64), rmp::encode::write_u64).unwrap(); + let mut b = Bytes::new(&out); + assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), Some(99)); + } + + #[test] + fn write_optional_none_writes_nil() { + let mut out = Vec::new(); + out.write_optional::(None, rmp::encode::write_u64).unwrap(); + let mut b = Bytes::new(&out); + assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), None); + } + + #[test] + fn write_optional_str() { + let mut out = Vec::new(); + out.write_optional(Some("hi"), rmp::encode::write_str).unwrap(); + let mut b = Bytes::new(&out); + assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), Some("hi".to_string())); + } +``` + +**Step 2: Run to verify failure** + +Run: `cargo test -p atuin-common --features rmp rmp::tests` +Expected: FAIL (`write_optional` not defined). + +**Step 3: Implement** + +Add the `RmpEncodeExt` trait and its blanket `impl` from **Appendix A**. + +**Step 4: Run to verify pass** + +Run: `cargo test -p atuin-common --features rmp rmp::tests` +Expected: PASS. +Run: `cargo clippy -p atuin-common --features rmp -- -D warnings` +Expected: PASS. + +**Step 5: Commit** +```bash +git add crates/atuin-common/src/rmp.rs +git commit -m "feat(common): add RmpEncodeExt write_optional" +``` + +--- + +### Task 6: Property tests (round-trips + never-panics) + +**Files:** +- Modify: `crates/atuin-common/src/rmp.rs` + +**Step 1: Write the proptests** + +Add to the `tests` module: +```rust + use proptest::prelude::*; + + proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + // A string survives an encode -> read_string round trip exactly. + #[test] + fn string_round_trips(s in r"(?s).*") { + let buf = enc(|v| rmp::encode::write_str(v, &s).unwrap()); + let mut b = Bytes::new(&buf); + prop_assert_eq!(b.read_string().unwrap(), s); + prop_assert!(b.remaining_slice().is_empty()); + } + + // Option survives write_optional -> read_optional. + #[test] + fn optional_u64_round_trips(v in proptest::option::of(any::())) { + let mut out = Vec::new(); + out.write_optional(v, rmp::encode::write_u64).unwrap(); + let mut b = Bytes::new(&out); + prop_assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), v); + } + + // Option survives the closure-based optional round trip. + #[test] + fn optional_string_round_trips(v in proptest::option::of(r"(?s).*")) { + let mut out = Vec::new(); + out.write_optional(v.as_deref(), rmp::encode::write_str).unwrap(); + let mut b = Bytes::new(&out); + prop_assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), v); + } + + // A full array record (len + fields + optional tail) round trips, and the + // cursor is exactly exhausted afterwards. + #[test] + fn record_round_trips( + id in r"[a-z]{0,16}", + ts in any::(), + deleted in proptest::option::of(any::()), + ) { + let mut out = Vec::new(); + rmp::encode::write_array_len(&mut out, 3).unwrap(); + rmp::encode::write_str(&mut out, &id).unwrap(); + rmp::encode::write_u64(&mut out, ts).unwrap(); + out.write_optional(deleted, rmp::encode::write_u64).unwrap(); + + let mut b = Bytes::new(&out); + prop_assert_eq!(b.expect_array_len(3).unwrap(), 3); + prop_assert_eq!(b.read_string().unwrap(), id); + prop_assert_eq!(b.read_with(rmp::decode::read_u64).unwrap(), ts); + prop_assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), deleted); + prop_assert!(b.expect_eof().is_ok()); + } + + // Reads never panic on arbitrary bytes — they return Err instead. + #[test] + fn reads_never_panic(raw in proptest::collection::vec(any::(), 0..64)) { + let mut b = Bytes::new(&raw); + let _ = b.read_string(); + let mut b = Bytes::new(&raw); + let _ = b.read_array_len(); + let mut b = Bytes::new(&raw); + let _ = b.read_optional(rmp::decode::read_u64); + } + } +``` + +**Step 2: Run** + +Run: `cargo test -p atuin-common --features rmp rmp::tests` +Expected: PASS. + +**Step 3: Commit** +```bash +git add crates/atuin-common/src/rmp.rs +git commit -m "test(common): property tests for rmp round-trips and panic-safety" +``` + +--- + +### Task 7: Migrate `history.rs` onto `atuin_common::rmp` + +**Files:** +- Modify: `crates/atuin-client/Cargo.toml` (enable the `rmp` feature on `atuin-common`) +- Modify: `crates/atuin-client/src/history.rs:1-13` (imports) and `:217-315` (serialize/deserialize) + +**Step 1: Enable the feature** + +In `crates/atuin-client/Cargo.toml`, under `[dependencies.atuin-common]`, add a features line: +```toml +[dependencies.atuin-common] +path = "../atuin-common" +features = ["rmp"] +``` +(Leave the `[dev-dependencies.atuin-common]` block as-is; cargo unifies features.) + +**Step 2: Swap the import** + +In `crates/atuin-client/src/history.rs`, replace: +```rust +use crate::utils::rmp::{DecodeError, EncodeError, read_optional, read_string, write_optional}; +``` +with: +```rust +use atuin_common::rmp::{DecodeError, EncodeError, RmpDecodeExt as _, RmpEncodeExt as _}; +``` +Keep the existing `use rmp::decode::{self, Bytes};` and `use rmp::encode;` lines — the raw crate is still used directly. + +**Step 3: Rewrite `serialize` (lines ~217-242)** + +Change the three `write_optional(&mut output, VALUE, WRITER)?` free-function calls to method calls: +```rust + output.write_optional( + self.deleted_at.map(|d| d.unix_timestamp_nanos() as u64), + encode::write_u64, + )?; + encode::write_str(&mut output, self.author.as_str())?; + output.write_optional(self.intent.as_deref(), encode::write_str)?; + output.write_optional(self.shell.as_deref(), encode::write_str)?; +``` + +**Step 4: Rewrite `deserialize` (lines ~244-297)** + +Replace free-function/`map_err(DecodeError::from)` calls with method calls. Note `read_string` becomes a method, so where it was passed as a bare fn to `read_optional`, use a closure: +```rust + let real_version = bytes.read_with(decode::read_u16)?; + // ... unchanged range check on real_version ... + + let nfields = bytes.read_array_len()?; + // ... unchanged version-dependent min/max range check + bail! ... + + let id = bytes.read_string()?; + let timestamp = bytes.read_with(decode::read_u64)?; + let duration = bytes.read_with(decode::read_int)?; + let exit = bytes.read_with(decode::read_int)?; + + let command = bytes.read_string()?; + let cwd = bytes.read_string()?; + let session = bytes.read_string()?; + let hostname = bytes.read_string()?; + let deleted_at = bytes.read_optional(decode::read_u64)?; + + let author = if version >= Version::One { + bytes.read_optional(|b| b.read_string())? + } else { + None + }; + // ... intent: same closure form ... + // ... shell: same closure form ... + + if version < Version::Two && !bytes.remaining_slice().is_empty() { + bail!("trailing bytes in encoded history. malformed") + } +``` +Keep the version-dependent trailing-bytes check as-is (it is conditional, not a plain `expect_eof`). + +**Step 5: Verify** + +Run: `cargo test -p atuin-client --lib history` +Expected: PASS (existing history serialize/deserialize round-trip tests still pass). +Run: `cargo clippy -p atuin-client -- -D warnings` +Expected: PASS. + +**Step 6: Commit** +```bash +git add crates/atuin-client/Cargo.toml crates/atuin-client/src/history.rs +git commit -m "refactor(client): use atuin_common::rmp in history serde" +``` + +--- + +### Task 8: Delete the old module + +**Files:** +- Delete: `crates/atuin-client/src/utils/rmp.rs` +- Modify: `crates/atuin-client/src/utils.rs:1` (remove `pub(crate) mod rmp;`) + +**Step 1: Confirm no remaining references** + +Run: `rg 'utils::rmp|utils/rmp' crates/` +Expected: no matches. + +**Step 2: Delete + unwire** +```bash +git rm crates/atuin-client/src/utils/rmp.rs +``` +Remove the `pub(crate) mod rmp;` line from `crates/atuin-client/src/utils.rs`. If that leaves `utils.rs` empty or `utils` unused, check `rg 'mod utils|utils::' crates/atuin-client/src` and clean up only if genuinely dead (do not remove other utils). + +**Step 3: Verify** + +Run: `cargo build -p atuin-client` +Expected: PASS. +Run: `cargo test -p atuin-client` +Expected: PASS. + +**Step 4: Commit** +```bash +git add -A +git commit -m "refactor(client): delete now-unused utils/rmp module" +``` + +**Stage A is complete here. The tree is green and the module is moved.** + +--- + +## Stage B — commonization payoff: migrate the raw-`rmp` callers + +Each Stage B task follows the same shape: enable `features = ["rmp"]` on the crate's `atuin-common` dependency (skip if already set), delete the local `error_report` fn, and replace the idioms: + +| Old idiom | New | +|---|---| +| `decode::read_x(&mut b).map_err(error_report)?` | `b.read_with(decode::read_x)?` | +| `let (s, rest) = read_str_from_slice(..).map_err(error_report)?; s.to_owned()` | `b.read_string()?` (owned) | +| `let n = read_array_len(..)?; ensure!(n == N, "...")` | `b.expect_array_len(N)?` | +| `if !bytes.is_empty() { bail!("trailing ...") }` | `b.expect_eof()?` (optionally `.wrap_err("")`) | + +Import `use atuin_common::rmp::{RmpDecodeExt as _, RmpEncodeExt as _};` (add `EncodeError`/`DecodeError` only where a type is named). Keep each file's `use rmp::{decode, encode};`. Normalize any `&mut &[u8]` reader to a `Bytes` cursor so the trait applies. + +> Behavior note: `expect_eof`/`expect_array_len` produce generic-but-informative messages (they include the actual counts/bytes). This replaces record-specific strings like "too many entries in v0 kv record". Where the specific context matters for debugging, append `.wrap_err("v0 kv record")?`. Confirm each crate's existing (de)serialize round-trip tests still pass after migration — the wire format must not change. + +### Task 9: `atuin-client` — `encryption.rs` and `history/store.rs` + +**Files:** +- Modify: `crates/atuin-client/src/encryption.rs:71-107` (`decode_key`) — replace the inline `.map_err(|err| eyre!("{err:?}"))` closures with `b.read_with(...)`; replace `ensure!(len == 32, ...)` on `read_array_len` with `b.expect_array_len(32)?` (the `Bin8`/`read_bin_len` branch keeps its own `ensure!` since `read_bin_len` isn't an array). Feature already enabled in Task 7. +- Modify: `crates/atuin-client/src/history/store.rs:73-114` (`deserialize`) — delete `error_report` (lines 76-78); use `b.read_with(decode::read_u8)`, `b.read_with(decode::read_bin_len)`; in the `Delete` arm replace `read_str_from_slice(..).map_err(error_report)?` + trailing check with `let id = bytes.read_string()?; bytes.expect_eof()?;`. + +**Step 1-2:** Make the edits above; keep the `Create` arm handing `bytes.remaining_slice()` to `History::deserialize` unchanged. + +**Step 3: Verify** + +Run: `cargo test -p atuin-client` +Expected: PASS. +Run: `cargo clippy -p atuin-client -- -D warnings` +Expected: PASS. + +**Step 4: Commit** +```bash +git add crates/atuin-client/src/encryption.rs crates/atuin-client/src/history/store.rs +git commit -m "refactor(client): use atuin_common::rmp in encryption and history store" +``` + +--- + +### Task 10: `atuin-scripts` — `store/record.rs` and `store/script.rs` + +**Files:** +- Modify: `crates/atuin-scripts/Cargo.toml` — add `features = ["rmp"]` to the `atuin-common` dependency. +- Modify: `crates/atuin-scripts/src/store/record.rs:49-91` — delete `error_report`; use `bytes.read_with(decode::read_u8)`, `bytes.read_with(decode::read_bin_len)`; in the delete arm, `bytes.read_string()?` instead of `read_str_from_slice`. +- Modify: `crates/atuin-scripts/src/store/script.rs:65-104` — this file uses `.unwrap()` throughout; convert to `?` via the trait: `bytes.expect_array_len(6)?`, `bytes.read_string()?` (×4 + the tags loop + script), and the tags nested `read_array_len` via `bytes.read_array_len()?`; replace the trailing `if !bytes.is_empty() { bail! }` with `bytes.expect_eof()?`. + +> `script.rs` currently `.unwrap()`s — migrating to `?` is a strict improvement (malformed data becomes an error instead of a panic). Keep the `ensure!(nfields == 6, ...)` semantics via `expect_array_len(6)`. + +**Step 1-2:** Make the edits. Normalize the alternating `Bytes`/`&[u8]` handling to a single `Bytes` cursor throughout (`read_string` advances it), removing the `Bytes::new(rest)` re-wrapping. + +**Step 3: Verify** + +Run: `cargo test -p atuin-scripts` +Expected: PASS. +Run: `cargo clippy -p atuin-scripts -- -D warnings` +Expected: PASS. + +**Step 4: Commit** +```bash +git add crates/atuin-scripts/Cargo.toml crates/atuin-scripts/src/store/record.rs crates/atuin-scripts/src/store/script.rs +git commit -m "refactor(scripts): use atuin_common::rmp in record/script serde" +``` + +--- + +### Task 11: `atuin-dotfiles` — `shell.rs`, `store.rs`, `store/var.rs` + +**Files:** +- Modify: `crates/atuin-dotfiles/Cargo.toml` — add `features = ["rmp"]` to `atuin-common`. +- Modify: `crates/atuin-dotfiles/src/shell.rs:44-76` (`Var::deserialize(bytes: &mut decode::Bytes)`) — delete `error_report`; `bytes.expect_array_len(3)?`; `bytes.read_string()?` for name/value; `bytes.read_with(decode::read_bool)?`; `bytes.expect_eof()?`. `Var::serialize(&mut Vec)` writer stays raw `encode::*`. +- Modify: `crates/atuin-dotfiles/src/store.rs:55-123` — delete `error_report`; `read_u8` via `read_with`; `expect_array_len(2)`/`expect_array_len(1)`; `read_string()`; `expect_eof()`. +- Modify: `crates/atuin-dotfiles/src/store/var.rs:49-99` — delete `error_report`; `read_u8` via `read_with`; delete-arm `expect_array_len(1)` + `read_string()` + `expect_eof()`; the create arm still delegates to `Var::deserialize(&mut bytes)`. + +**Step 1-2:** Make the edits. `shell.rs`'s `&mut decode::Bytes` receiver is exactly what `RmpDecodeExt for Bytes` needs — no signature change. + +**Step 3: Verify** + +Run: `cargo test -p atuin-dotfiles` +Expected: PASS. +Run: `cargo clippy -p atuin-dotfiles -- -D warnings` +Expected: PASS. + +**Step 4: Commit** +```bash +git add crates/atuin-dotfiles/Cargo.toml crates/atuin-dotfiles/src/shell.rs crates/atuin-dotfiles/src/store.rs crates/atuin-dotfiles/src/store/var.rs +git commit -m "refactor(dotfiles): use atuin_common::rmp across var/alias serde" +``` + +--- + +### Task 12: `atuin-kv` — `store/record.rs` + +**Files:** +- Modify: `crates/atuin-kv/Cargo.toml` — add `features = ["rmp"]` to `atuin-common`. +- Modify: `crates/atuin-kv/src/store/record.rs:36-102` — delete `error_report`; both version arms use `expect_array_len(3)` / `expect_array_len(4)`, `read_string()`, `expect_eof()`. Critically, normalize the `v1` branch's `read_bool` on a raw `&mut &[u8]` (line 78) to operate on the `Bytes` cursor: `let has_value = bytes.read_with(decode::read_bool)?;` then continue reading from the same `bytes` cursor (`if has_value { Some(bytes.read_string()?) } else { None }`), and `bytes.expect_eof()?`. + +**Step 1-2:** Make the edits. This removes the only `&mut &[u8]` reader in the codebase, unifying on `Bytes`. + +**Step 3: Verify** + +Run: `cargo test -p atuin-kv` +Expected: PASS. +Run: `cargo clippy -p atuin-kv -- -D warnings` +Expected: PASS. + +**Step 4: Commit** +```bash +git add crates/atuin-kv/Cargo.toml crates/atuin-kv/src/store/record.rs +git commit -m "refactor(kv): use atuin_common::rmp in record serde" +``` + +--- + +### Task 13: Full-workspace verification + +**Step 1: Confirm the boilerplate is gone** + +Run: `rg 'fn error_report' crates/` +Expected: no matches. +Run: `rg 'map_err\(\|err\| eyre!\("\{err:\?\}"\)\)' crates/` +Expected: no matches. + +**Step 2: Build, test, lint the whole workspace** + +Run: `cargo build --workspace` +Expected: PASS. +Run: `cargo test --workspace` +Expected: PASS. +Run: `cargo clippy --workspace --all-targets -- -D warnings` +Expected: PASS. + +**Step 3: Verify the feature-off path still compiles** + +Run: `cargo build -p atuin-common` (no `rmp` feature) +Expected: PASS (module excluded). + +**Step 4: Commit any final touch-ups** +```bash +git add -A +git commit -m "chore: verify workspace after rmp commonization" +``` + +--- + +## Appendix A — Complete target `crates/atuin-common/src/rmp.rs` (module code, no tests) + +```rust +//! MessagePack encode/decode helpers built on [`rmp`]. +//! +//! `rmp`'s own error types are awkward: some don't implement [`Display`] for +//! every `E`, none say which variant they are, and the decode cursor offers no +//! owned-string read, no nil-aware optional read, and no structural checks +//! (array length, end-of-input). This module fills those gaps with two +//! extension traits — [`RmpDecodeExt`] for reading a [`Bytes`] cursor and +//! [`RmpEncodeExt`] for writing — plus [`DecodeError`] / [`EncodeError`], which +//! carry legible messages and convert cleanly into [`eyre::Report`]. +//! +//! Because every decode read returns [`DecodeError`], a decoder can use `?` +//! throughout and convert once at the boundary, rather than hand-rolling a +//! `map_err` at every read. +//! +//! [`Display`]: std::fmt::Display + +use rmp::Marker; +use rmp::decode::bytes::{Bytes, BytesReadError}; +use rmp::decode::{self, DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError}; +use rmp::encode::{self, RmpWrite, RmpWriteErr, ValueWriteError}; + +/// An error encountered while encoding a MessagePack value. +/// +/// Wraps [`ValueWriteError`] with a message that names the failing write and its +/// inner I/O error — neither of which `rmp`'s own [`Display`] reports. Implements +/// [`std::error::Error`], so it converts into [`eyre::Report`] with `?`. +/// +/// [`Display`]: std::fmt::Display +#[derive(Debug, derive_more::Display, derive_more::From, thiserror::Error)] +#[display("could not write MessagePack value: {_0:?}")] +pub struct EncodeError(ValueWriteError); + +/// An error encountered while decoding a MessagePack value. +/// +/// Wraps the three error types `rmp`'s decode functions return, and adds two +/// structural variants for checks `rmp` does not perform itself: +/// [`UnexpectedArrayLen`](Self::UnexpectedArrayLen) and +/// [`TrailingBytes`](Self::TrailingBytes). +/// +/// Converts into [`eyre::Report`] via a manual `From` rather than a +/// [`std::error::Error`] impl, because a [`DecodeError`] is not, in general, +/// `'static`. +/// +/// [`Display`]: std::fmt::Display +#[derive(Debug, derive_more::Display)] +pub enum DecodeError<'a, E: RmpReadErr = BytesReadError> { + /// The next value was not a valid UTF-8 string. + #[display("could not decode MessagePack string: {_0:?}")] + DecodeString(DecodeStringError<'a, E>), + /// The next value was not the expected number. + #[display("could not decode MessagePack number: {_0:?}")] + NumValueRead(NumValueReadError), + /// The next value could not be decoded. + #[display("could not decode MessagePack value: {_0:?}")] + ValueRead(ValueReadError), + /// An array marker did not have the expected length. + #[display("expected a MessagePack array of length {expected}, found {actual}")] + UnexpectedArrayLen { expected: u32, actual: u32 }, + /// Input remained after a value was fully decoded. + #[display("{remaining} trailing byte(s) after decoding MessagePack value")] + TrailingBytes { remaining: usize }, +} + +impl<'a, E: RmpReadErr> From> for DecodeError<'a, E> { + fn from(e: DecodeStringError<'a, E>) -> Self { + Self::DecodeString(e) + } +} + +impl From> for DecodeError<'_, E> { + fn from(e: NumValueReadError) -> Self { + Self::NumValueRead(e) + } +} + +impl From> for DecodeError<'_, E> { + fn from(e: ValueReadError) -> Self { + Self::ValueRead(e) + } +} + +impl DecodeError<'_, E> { + /// If this is a type mismatch, the [`Marker`] found instead of the expected + /// type. [`RmpDecodeExt::read_optional`] uses this to recognise nil. + pub fn type_mismatch(&self) -> Option { + match self { + Self::DecodeString(DecodeStringError::TypeMismatch(m)) + | Self::NumValueRead(NumValueReadError::TypeMismatch(m)) + | Self::ValueRead(ValueReadError::TypeMismatch(m)) => Some(*m), + _ => None, + } + } +} + +impl From> for eyre::Report { + fn from(e: DecodeError<'_, E>) -> Self { + eyre::eyre!("{e}") + } +} + +/// Reading helpers for a MessagePack [`Bytes`] cursor. +/// +/// Every method reports failures as [`DecodeError`], so decoders use `?` +/// throughout and convert once at the boundary. +pub trait RmpDecodeExt<'a> { + /// Run an `rmp` decode function, converting its error into [`DecodeError`]. + /// + /// Lets raw `rmp::decode` functions compose with `?`: + /// `bytes.read_with(rmp::decode::read_u64)?`. + fn read_with(&mut self, read: F) -> Result> + where + F: FnOnce(&mut Self) -> Result, + E: Into>; + + /// Read an owned [`String`]. + fn read_string(&mut self) -> Result>; + + /// Read a value that may be encoded as nil, returning [`None`] for nil. + /// + /// `read` decodes a `T`; if it fails specifically because it found + /// [`Marker::Null`], this yields [`None`] with the cursor left just past the + /// nil. Any other error is forwarded unchanged. + fn read_optional(&mut self, read: F) -> Result, DecodeError<'a>> + where + F: FnOnce(&mut Self) -> Result, + E: Into>; + + /// Read an array-length marker. + fn read_array_len(&mut self) -> Result>; + + /// Read an array-length marker and require it to equal `expected`. + /// + /// Returns [`DecodeError::UnexpectedArrayLen`] otherwise. For a + /// forward-compatible field count, use [`read_array_len`](Self::read_array_len) + /// and range-check the value yourself. + fn expect_array_len(&mut self, expected: u32) -> Result>; + + /// Succeed only if the cursor is at end-of-input, else + /// [`DecodeError::TrailingBytes`] — the standard malformed-record guard after + /// a fixed set of fields. + fn expect_eof(&self) -> Result<(), DecodeError<'a>>; +} + +impl<'a> RmpDecodeExt<'a> for Bytes<'a> { + fn read_with(&mut self, read: F) -> Result> + where + F: FnOnce(&mut Self) -> Result, + E: Into>, + { + read(self).map_err(Into::into) + } + + fn read_string(&mut self) -> Result> { + let slice = self.remaining_slice(); + let (string, rest) = match decode::read_str_from_slice(slice) { + Ok(pair) => pair, + Err(e) => { + if let DecodeStringError::TypeMismatch(_) = e { + // rmp's decode functions consume the marker byte on a type + // mismatch; do the same so `read_optional` can detect nil. + self.read_u8() + .expect("TypeMismatch implies the stream contains a marker byte"); + } + return Err(e.into()); + } + }; + *self = Bytes::new(rest); + Ok(string.into()) + } + + fn read_optional(&mut self, read: F) -> Result, DecodeError<'a>> + where + F: FnOnce(&mut Self) -> Result, + E: Into>, + { + match read(self) { + Ok(v) => Ok(Some(v)), + Err(e) => { + let e = e.into(); + if let Some(Marker::Null) = e.type_mismatch() { + Ok(None) + } else { + Err(e) + } + } + } + } + + fn read_array_len(&mut self) -> Result> { + decode::read_array_len(self).map_err(Into::into) + } + + fn expect_array_len(&mut self, expected: u32) -> Result> { + let actual = self.read_array_len()?; + if actual == expected { + Ok(actual) + } else { + Err(DecodeError::UnexpectedArrayLen { expected, actual }) + } + } + + fn expect_eof(&self) -> Result<(), DecodeError<'a>> { + let remaining = self.remaining_slice().len(); + if remaining == 0 { + Ok(()) + } else { + Err(DecodeError::TrailingBytes { remaining }) + } + } +} + +/// Writing helpers for a MessagePack output buffer. +pub trait RmpEncodeExt: RmpWrite { + /// Write an optional value, encoding [`None`] as [`Marker::Null`]. + /// + /// The mirror of [`RmpDecodeExt::read_optional`]: `write` encodes the inner + /// value when `value` is [`Some`]. + fn write_optional( + &mut self, + value: Option, + write: F, + ) -> Result<(), EncodeError> + where + F: FnOnce(&mut Self, T) -> Result<(), ValueWriteError>; +} + +impl RmpEncodeExt for W { + fn write_optional( + &mut self, + value: Option, + write: F, + ) -> Result<(), EncodeError> + where + F: FnOnce(&mut Self, T) -> Result<(), ValueWriteError>, + { + match value { + Some(v) => write(self, v).map_err(EncodeError::from), + None => encode::write_nil(self) + .map_err(|e| EncodeError::from(ValueWriteError::InvalidMarkerWrite(e))), + } + } +} +``` + +> **If the build disagrees with Appendix A**, trust the compiler and the *old* module's proven signatures (`crates/atuin-client/src/utils/rmp.rs` in git history) over this appendix — particularly the exact `RmpWrite::Error` / `ValueWriteError` bounds. The old `read_string`/`read_optional`/`write_optional` bodies are known-good; the only substantive additions here are the two structural `DecodeError` variants and the `read_with`/`read_array_len`/`expect_array_len`/`expect_eof` methods. From 526a1cb6b448c100bcac6023f2a7a40ac9e64262 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 12:56:05 -0700 Subject: [PATCH 11/27] refactor(client): use atuin_common::rmp in encryption and history store --- crates/atuin-client/src/encryption.rs | 11 ++++++----- crates/atuin-client/src/history/store.rs | 21 ++++++--------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/crates/atuin-client/src/encryption.rs b/crates/atuin-client/src/encryption.rs index 1cdf9ee5516..ea6948350c4 100644 --- a/crates/atuin-client/src/encryption.rs +++ b/crates/atuin-client/src/encryption.rs @@ -13,10 +13,12 @@ use std::io::prelude::*; use base64::prelude::{BASE64_STANDARD, Engine}; pub use crypto_secretbox::Key; use crypto_secretbox::{KeyInit, XSalsa20Poly1305, aead::OsRng}; -use eyre::{Context, Result, bail, ensure, eyre}; +use eyre::{Context, Result, bail, ensure}; use fs_err as fs; use rmp::Marker; +use atuin_common::rmp::RmpDecodeExt as _; + use crate::settings::Settings; pub fn generate_encoded_key() -> Result<(Key, String)> { @@ -84,19 +86,18 @@ pub fn decode_key(key: String) -> Result { match Marker::from_u8(buf[0]) { Marker::Bin8 => { - let len = decode::read_bin_len(&mut bytes).map_err(|err| eyre!("{err:?}"))?; + let len = bytes.read_with(decode::read_bin_len)?; ensure!(len == 32, "encryption key is not the correct size"); let key = <[u8; 32]>::try_from(bytes.remaining_slice()) .context("could not decode encryption key")?; Ok(key.into()) } Marker::Array16 => { - let len = decode::read_array_len(&mut bytes).map_err(|err| eyre!("{err:?}"))?; - ensure!(len == 32, "encryption key is not the correct size"); + bytes.expect_array_len(32)?; let mut key = Key::default(); for i in &mut key { - *i = rmp::decode::read_int(&mut bytes).map_err(|err| eyre!("{err:?}"))?; + *i = bytes.read_with(decode::read_int)?; } Ok(key) } diff --git a/crates/atuin-client/src/history/store.rs b/crates/atuin-client/src/history/store.rs index ba538aa4bad..94a4d09ab5b 100644 --- a/crates/atuin-client/src/history/store.rs +++ b/crates/atuin-client/src/history/store.rs @@ -10,6 +10,7 @@ use crate::{ record::{encryption::PASETO_V4, sqlite_store::SqliteStore, store::Store}, }; use atuin_common::record::{DecryptedData, Host, HostId, Record, RecordId, RecordIdx}; +use atuin_common::rmp::RmpDecodeExt as _; use super::{HISTORY_TAG, History, HistoryId, Version}; @@ -73,20 +74,16 @@ impl HistoryRecord { pub fn deserialize(bytes: &DecryptedData, version: &str) -> Result { use rmp::decode; - fn error_report(err: E) -> eyre::Report { - eyre!("{err:?}") - } - let mut bytes = Bytes::new(&bytes.0); - let record_type = decode::read_u8(&mut bytes).map_err(error_report)?; + let record_type = bytes.read_with(decode::read_u8)?; match record_type { // 0 -> HistoryRecord::Create 0 => { // not super useful to us atm, but perhaps in the future // written by write_bin above - let _ = decode::read_bin_len(&mut bytes).map_err(error_report)?; + let _ = bytes.read_with(decode::read_bin_len)?; let record = History::deserialize(bytes.remaining_slice(), version)?; @@ -95,16 +92,10 @@ impl HistoryRecord { // 1 -> HistoryRecord::Delete 1 => { - let bytes = bytes.remaining_slice(); - let (id, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; - - if !bytes.is_empty() { - bail!( - "trailing bytes decoding HistoryRecord::Delete - malformed? got {bytes:?}" - ); - } + let id = bytes.read_string()?; + bytes.expect_eof()?; - Ok(HistoryRecord::Delete(id.to_string().into())) + Ok(HistoryRecord::Delete(id.into())) } n => { From c30019ccfb78a7c5d18cec061ae925e19c8f2c71 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 13:00:38 -0700 Subject: [PATCH 12/27] refactor(scripts): use atuin_common::rmp in record/script serde --- crates/atuin-scripts/Cargo.toml | 2 +- crates/atuin-scripts/src/store/record.rs | 16 +++----- crates/atuin-scripts/src/store/script.rs | 48 ++++++++++-------------- 3 files changed, 26 insertions(+), 40 deletions(-) diff --git a/crates/atuin-scripts/Cargo.toml b/crates/atuin-scripts/Cargo.toml index 80408679377..fb2e8cce2ee 100644 --- a/crates/atuin-scripts/Cargo.toml +++ b/crates/atuin-scripts/Cargo.toml @@ -15,7 +15,7 @@ readme.workspace = true [dependencies] atuin-client = { path = "../atuin-client", version = "18.17.1" } -atuin-common = { path = "../atuin-common", version = "18.17.1" } +atuin-common = { path = "../atuin-common", version = "18.17.1", features = ["rmp"] } derive_more = { workspace = true } tracing = { workspace = true } diff --git a/crates/atuin-scripts/src/store/record.rs b/crates/atuin-scripts/src/store/record.rs index 4c925be3a2f..bc3f12d6e64 100644 --- a/crates/atuin-scripts/src/store/record.rs +++ b/crates/atuin-scripts/src/store/record.rs @@ -47,38 +47,34 @@ impl ScriptRecord { } pub fn deserialize(data: &DecryptedData, version: &str) -> Result { + use atuin_common::rmp::RmpDecodeExt as _; use rmp::decode; - fn error_report(err: E) -> eyre::Report { - eyre!("{err:?}") - } - match version { SCRIPT_VERSION => { let mut bytes = decode::Bytes::new(&data.0); - let record_type = decode::read_u8(&mut bytes).map_err(error_report)?; + let record_type = bytes.read_with(decode::read_u8)?; match record_type { // create 0 => { // written by encode::write_bin above - let _ = decode::read_bin_len(&mut bytes).map_err(error_report)?; + let _ = bytes.read_with(decode::read_bin_len)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Create(script)) } // delete 1 => { - let bytes = bytes.remaining_slice(); - let (id, _) = decode::read_str_from_slice(bytes).map_err(error_report)?; - Ok(ScriptRecord::Delete(Uuid::parse_str(id)?)) + let id = bytes.read_string()?; + Ok(ScriptRecord::Delete(Uuid::parse_str(&id)?)) } // update 2 => { // written by encode::write_bin above - let _ = decode::read_bin_len(&mut bytes).map_err(error_report)?; + let _ = bytes.read_with(decode::read_bin_len)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Update(script)) } diff --git a/crates/atuin-scripts/src/store/script.rs b/crates/atuin-scripts/src/store/script.rs index af180320844..1c05a008cb0 100644 --- a/crates/atuin-scripts/src/store/script.rs +++ b/crates/atuin-scripts/src/store/script.rs @@ -1,11 +1,8 @@ use atuin_common::record::DecryptedData; -use eyre::{Result, bail, ensure}; +use eyre::Result; use uuid::Uuid; -use rmp::{ - decode::{self, Bytes}, - encode, -}; +use rmp::{decode, encode}; use typed_builder::TypedBuilder; pub const SCRIPT_VERSION: &str = "v0"; @@ -63,43 +60,36 @@ impl Script { } pub fn deserialize(bytes: &[u8]) -> Result { - let mut bytes = decode::Bytes::new(bytes); - let nfields = decode::read_array_len(&mut bytes).unwrap(); - - ensure!(nfields == 6, "too many entries in v0 script record"); + use atuin_common::rmp::RmpDecodeExt as _; - let bytes = bytes.remaining_slice(); + let mut bytes = decode::Bytes::new(bytes); - let (id, bytes) = decode::read_str_from_slice(bytes).unwrap(); - let (name, bytes) = decode::read_str_from_slice(bytes).unwrap(); - let (description, bytes) = decode::read_str_from_slice(bytes).unwrap(); - let (shebang, bytes) = decode::read_str_from_slice(bytes).unwrap(); + bytes.expect_array_len(6)?; - let mut bytes = Bytes::new(bytes); - let tags_len = decode::read_array_len(&mut bytes).unwrap(); + let id = bytes.read_string()?; + let name = bytes.read_string()?; + let description = bytes.read_string()?; + let shebang = bytes.read_string()?; - let mut bytes = bytes.remaining_slice(); + let tags_len = bytes.read_array_len()?; let mut tags = Vec::new(); for _ in 0..tags_len { - let (tag, remaining) = decode::read_str_from_slice(bytes).unwrap(); - tags.push(tag.to_owned()); - bytes = remaining; + let tag = bytes.read_string()?; + tags.push(tag); } - let (script, bytes) = decode::read_str_from_slice(bytes).unwrap(); + let script = bytes.read_string()?; - if !bytes.is_empty() { - bail!("trailing bytes in encoded script record. malformed") - } + bytes.expect_eof()?; Ok(Script { - id: Uuid::parse_str(id).unwrap(), - name: name.to_owned(), - description: description.to_owned(), - shebang: shebang.to_owned(), + id: Uuid::parse_str(&id)?, + name, + description, + shebang, tags, - script: script.to_owned(), + script, }) } } From af8d4bcea341c39c4887850b41a3dbaf03918871 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 13:05:20 -0700 Subject: [PATCH 13/27] refactor(dotfiles): use atuin_common::rmp across var/alias serde --- crates/atuin-dotfiles/Cargo.toml | 2 +- crates/atuin-dotfiles/src/shell.rs | 35 +++++----------- crates/atuin-dotfiles/src/store.rs | 55 +++++++------------------- crates/atuin-dotfiles/src/store/var.rs | 26 ++++-------- 4 files changed, 32 insertions(+), 86 deletions(-) diff --git a/crates/atuin-dotfiles/Cargo.toml b/crates/atuin-dotfiles/Cargo.toml index ab6be280305..53f7a213f6b 100644 --- a/crates/atuin-dotfiles/Cargo.toml +++ b/crates/atuin-dotfiles/Cargo.toml @@ -14,7 +14,7 @@ readme.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -atuin-common = { path = "../atuin-common", version = "18.17.1" } +atuin-common = { path = "../atuin-common", version = "18.17.1", features = ["rmp"] } atuin-client = { path = "../atuin-client", version = "18.17.1" } derive_more = { workspace = true } diff --git a/crates/atuin-dotfiles/src/shell.rs b/crates/atuin-dotfiles/src/shell.rs index 73a9ce8cba3..02b8fbdaeb9 100644 --- a/crates/atuin-dotfiles/src/shell.rs +++ b/crates/atuin-dotfiles/src/shell.rs @@ -1,7 +1,8 @@ -use eyre::{Result, ensure, eyre}; +use eyre::Result; use rmp::{decode, encode}; use serde::Serialize; +use atuin_common::rmp::RmpDecodeExt as _; use atuin_common::shell::{Shell, ShellError}; use crate::store::AliasStore; @@ -42,35 +43,17 @@ impl Var { } pub fn deserialize(bytes: &mut decode::Bytes) -> Result { - fn error_report(err: E) -> eyre::Report { - eyre!("{err:?}") - } - - let nfields = decode::read_array_len(bytes).map_err(error_report)?; - - ensure!( - nfields == 3, - "too many entries in v0 dotfiles env create record, got {}, expected {}", - nfields, - 3 - ); + bytes.expect_array_len(3)?; - let bytes = bytes.remaining_slice(); + let name = bytes.read_string()?; + let value = bytes.read_string()?; + let export = bytes.read_with(decode::read_bool)?; - let (key, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; - let (value, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; - - let mut bytes = decode::Bytes::new(bytes); - let export = decode::read_bool(&mut bytes).map_err(error_report)?; - - ensure!( - bytes.remaining_slice().is_empty(), - "trailing bytes in encoded dotfiles env record, malformed" - ); + bytes.expect_eof()?; Ok(Var { - name: key.to_owned(), - value: value.to_owned(), + name, + value, export, }) } diff --git a/crates/atuin-dotfiles/src/store.rs b/crates/atuin-dotfiles/src/store.rs index a3b9a7546c1..c915fdf45e7 100644 --- a/crates/atuin-dotfiles/src/store.rs +++ b/crates/atuin-dotfiles/src/store.rs @@ -6,8 +6,9 @@ use atuin_client::record::sqlite_store::SqliteStore; // While we will support a range of shell config, I'd rather have a larger number of small records // + stores, rather than one mega config store. use atuin_common::record::{DecryptedData, Host, HostId}; +use atuin_common::rmp::RmpDecodeExt as _; use atuin_common::utils::unquote; -use eyre::{Result, bail, ensure, eyre}; +use eyre::{Result, bail, eyre}; use atuin_client::record::encryption::PASETO_V4; use atuin_client::record::store::Store; @@ -55,60 +56,34 @@ impl AliasRecord { pub fn deserialize(data: &DecryptedData, version: &str) -> Result { use rmp::decode; - fn error_report(err: E) -> eyre::Report { - eyre!("{err:?}") - } - match version { CONFIG_SHELL_ALIAS_VERSION => { let mut bytes = decode::Bytes::new(&data.0); - let record_type = decode::read_u8(&mut bytes).map_err(error_report)?; + let record_type = bytes.read_with(decode::read_u8)?; match record_type { // create 0 => { - let nfields = decode::read_array_len(&mut bytes).map_err(error_report)?; - ensure!( - nfields == 2, - "too many entries in v0 shell alias create record" - ); - - let bytes = bytes.remaining_slice(); - - let (key, bytes) = - decode::read_str_from_slice(bytes).map_err(error_report)?; - let (value, bytes) = - decode::read_str_from_slice(bytes).map_err(error_report)?; - - if !bytes.is_empty() { - bail!("trailing bytes in encoded shell alias record. malformed") - } - - Ok(AliasRecord::Create(Alias { - name: key.to_owned(), - value: value.to_owned(), - })) + bytes.expect_array_len(2)?; + + let name = bytes.read_string()?; + let value = bytes.read_string()?; + + bytes.expect_eof()?; + + Ok(AliasRecord::Create(Alias { name, value })) } // delete 1 => { - let nfields = decode::read_array_len(&mut bytes).map_err(error_report)?; - ensure!( - nfields == 1, - "too many entries in v0 shell alias delete record" - ); - - let bytes = bytes.remaining_slice(); + bytes.expect_array_len(1)?; - let (key, bytes) = - decode::read_str_from_slice(bytes).map_err(error_report)?; + let key = bytes.read_string()?; - if !bytes.is_empty() { - bail!("trailing bytes in encoded shell alias record. malformed") - } + bytes.expect_eof()?; - Ok(AliasRecord::Delete(key.to_owned())) + Ok(AliasRecord::Delete(key)) } n => { diff --git a/crates/atuin-dotfiles/src/store/var.rs b/crates/atuin-dotfiles/src/store/var.rs index 5fc67328be6..de3216de75b 100644 --- a/crates/atuin-dotfiles/src/store/var.rs +++ b/crates/atuin-dotfiles/src/store/var.rs @@ -6,7 +6,8 @@ use std::collections::BTreeMap; use atuin_client::record::sqlite_store::SqliteStore; use atuin_common::record::{DecryptedData, Host, HostId}; -use eyre::{Result, bail, ensure, eyre}; +use atuin_common::rmp::RmpDecodeExt as _; +use eyre::{Result, bail, eyre}; use atuin_client::record::encryption::PASETO_V4; use atuin_client::record::store::Store; @@ -49,15 +50,11 @@ impl VarRecord { pub fn deserialize(data: &DecryptedData, version: &str) -> Result { use rmp::decode; - fn error_report(err: E) -> eyre::Report { - eyre!("{err:?}") - } - match version { DOTFILES_VAR_VERSION => { let mut bytes = decode::Bytes::new(&data.0); - let record_type = decode::read_u8(&mut bytes).map_err(error_report)?; + let record_type = bytes.read_with(decode::read_u8)?; match record_type { // create @@ -68,22 +65,13 @@ impl VarRecord { // delete 1 => { - let nfields = decode::read_array_len(&mut bytes).map_err(error_report)?; - ensure!( - nfields == 1, - "too many entries in v0 dotfiles var delete record" - ); - - let bytes = bytes.remaining_slice(); + bytes.expect_array_len(1)?; - let (key, bytes) = - decode::read_str_from_slice(bytes).map_err(error_report)?; + let key = bytes.read_string()?; - if !bytes.is_empty() { - bail!("trailing bytes in encoded dotfiles var record. malformed") - } + bytes.expect_eof()?; - Ok(VarRecord::Delete(key.to_owned())) + Ok(VarRecord::Delete(key)) } n => { From 424c1b359cb09412e5576e395e2334555651d49b Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 13:08:47 -0700 Subject: [PATCH 14/27] test(dotfiles): round-trip delete-record deserialize --- crates/atuin-dotfiles/src/store.rs | 10 ++++++++++ crates/atuin-dotfiles/src/store/var.rs | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/crates/atuin-dotfiles/src/store.rs b/crates/atuin-dotfiles/src/store.rs index c915fdf45e7..53486cd2c53 100644 --- a/crates/atuin-dotfiles/src/store.rs +++ b/crates/atuin-dotfiles/src/store.rs @@ -356,6 +356,16 @@ mod tests { assert_eq!(decoded, record); } + #[test] + fn encode_decode_delete() { + let record = AliasRecord::Delete("k".to_owned()); + + let encoded = record.serialize().unwrap(); + let decoded = AliasRecord::deserialize(&encoded, CONFIG_SHELL_ALIAS_VERSION).unwrap(); + + assert_eq!(decoded, record); + } + #[tokio::test] async fn build_aliases() { let store = SqliteStore::new(":memory:", test_local_timeout()) diff --git a/crates/atuin-dotfiles/src/store/var.rs b/crates/atuin-dotfiles/src/store/var.rs index de3216de75b..3d329a32e53 100644 --- a/crates/atuin-dotfiles/src/store/var.rs +++ b/crates/atuin-dotfiles/src/store/var.rs @@ -409,6 +409,16 @@ mod tests { assert_eq!(decoded, record); } + #[test] + fn encode_decode_delete() { + let record = VarRecord::Delete("BEEP".to_owned()); + + let encoded = record.serialize().unwrap(); + let decoded = VarRecord::deserialize(&encoded, DOTFILES_VAR_VERSION).unwrap(); + + assert_eq!(decoded, record); + } + #[rstest] // Simple values should not be quoted #[case::simple("simple", "simple")] From 0171b3dba3129ddb441acd38153907a8fdbae374 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 13:09:57 -0700 Subject: [PATCH 15/27] refactor(kv): use atuin_common::rmp in record serde --- crates/atuin-kv/Cargo.toml | 2 +- crates/atuin-kv/src/store/record.rs | 57 ++++++++++------------------- 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/crates/atuin-kv/Cargo.toml b/crates/atuin-kv/Cargo.toml index 7f0990c08ec..9e74f848877 100644 --- a/crates/atuin-kv/Cargo.toml +++ b/crates/atuin-kv/Cargo.toml @@ -15,7 +15,7 @@ readme.workspace = true [dependencies] atuin-client = { path = "../atuin-client", version = "18.17.1" } -atuin-common = { path = "../atuin-common", version = "18.17.1" } +atuin-common = { path = "../atuin-common", version = "18.17.1", features = ["rmp"] } derive_more = { workspace = true } tracing = { workspace = true } diff --git a/crates/atuin-kv/src/store/record.rs b/crates/atuin-kv/src/store/record.rs index 3725417636c..f41e03ccaa2 100644 --- a/crates/atuin-kv/src/store/record.rs +++ b/crates/atuin-kv/src/store/record.rs @@ -1,5 +1,6 @@ use atuin_common::record::DecryptedData; -use eyre::{Result, bail, ensure, eyre}; +use atuin_common::rmp::RmpDecodeExt as _; +use eyre::{Result, bail}; use typed_builder::TypedBuilder; pub const KV_VERSION: &str = "v1"; @@ -36,62 +37,44 @@ impl KvRecord { pub fn deserialize(data: &DecryptedData, version: &str) -> Result { use rmp::decode; - fn error_report(err: E) -> eyre::Report { - eyre!("{err:?}") - } - match version { "v0" => { let mut bytes = decode::Bytes::new(&data.0); - let nfields = decode::read_array_len(&mut bytes).map_err(error_report)?; - ensure!(nfields == 3, "too many entries in v0 kv record"); + bytes.expect_array_len(3)?; - let bytes = bytes.remaining_slice(); + let namespace = bytes.read_string()?; + let key = bytes.read_string()?; + let value = bytes.read_string()?; - let (namespace, bytes) = - decode::read_str_from_slice(bytes).map_err(error_report)?; - let (key, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; - let (value, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; - - if !bytes.is_empty() { - bail!("trailing bytes in encoded kvrecord. malformed") - } + bytes.expect_eof()?; Ok(KvRecord { - namespace: namespace.to_owned(), - key: key.to_owned(), - value: Some(value.to_owned()), + namespace, + key, + value: Some(value), }) } KV_VERSION => { let mut bytes = decode::Bytes::new(&data.0); - let nfields = decode::read_array_len(&mut bytes).map_err(error_report)?; - ensure!(nfields == 4, "too many entries in v1 kv record"); - - let bytes = bytes.remaining_slice(); + bytes.expect_array_len(4)?; - let (namespace, bytes) = - decode::read_str_from_slice(bytes).map_err(error_report)?; - let (key, mut bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; - let has_value = decode::read_bool(&mut bytes).map_err(error_report)?; + let namespace = bytes.read_string()?; + let key = bytes.read_string()?; + let has_value = bytes.read_with(decode::read_bool)?; - let (value, bytes) = if has_value { - let (value, bytes) = - decode::read_str_from_slice(bytes).map_err(error_report)?; - (Some(value.to_owned()), bytes) + let value = if has_value { + Some(bytes.read_string()?) } else { - (None, bytes) + None }; - if !bytes.is_empty() { - bail!("trailing bytes in encoded kvrecord. malformed") - } + bytes.expect_eof()?; Ok(KvRecord { - namespace: namespace.to_owned(), - key: key.to_owned(), + namespace, + key, value, }) } From 1a0d71576cd719e0a142622c1851643b768404b7 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 22:56:10 -0700 Subject: [PATCH 16/27] deslop --- docs/plans/2026-07-07-sync-bench-infra.md | 486 -------- ...026-07-18-commonize-rmp-msgpack-helpers.md | 1014 ----------------- 2 files changed, 1500 deletions(-) delete mode 100644 docs/plans/2026-07-07-sync-bench-infra.md delete mode 100644 docs/plans/2026-07-18-commonize-rmp-msgpack-helpers.md diff --git a/docs/plans/2026-07-07-sync-bench-infra.md b/docs/plans/2026-07-07-sync-bench-infra.md deleted file mode 100644 index 966d9d3993f..00000000000 --- a/docs/plans/2026-07-07-sync-bench-infra.md +++ /dev/null @@ -1,486 +0,0 @@ -# Sync Benchmark Infrastructure Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Build benchmark infrastructure that exercises record sync over real HTTP, so page size changes (100→1000) show measurable round-trip savings. - -**Architecture:** The benchmark starts an in-process axum server backed by an in-memory `Database` implementation (no Postgres needed). A real `reqwest`-based `Client` talks to it over TCP loopback. The benchmark uploads N records, then measures the wall-clock cost of downloading them all at varying page sizes. This isolates HTTP round-trip overhead — the thing the page size change actually optimizes. - -The in-memory `Database` lives in a new crate `atuin-bench-support` to keep it reusable across bench targets. The benchmark itself lives in `crates/atuin/benches/` since the top-level `atuin` crate already has `atuin-server`, `atuin-server-database`, and `atuin-server-postgres` as dev-dependencies. - -**Tech Stack:** Rust, divan, axum (in-process), reqwest, tokio, `atuin-server`'s `launch_with_tcp_listener` - ---- - -### Task 1: Create `atuin-bench-support` crate with in-memory Database - -The `Database` trait (`atuin-server-database/src/lib.rs:114`) has ~20 methods. Only 8 are needed for record sync benchmarks. The rest return `DbError::NotFound` or empty results. - -**Files:** -- Create: `crates/atuin-bench-support/Cargo.toml` -- Create: `crates/atuin-bench-support/src/lib.rs` -- Modify: `Cargo.toml` (workspace members) - -**Step 1: Add crate to workspace** - -In the root `Cargo.toml`, add `"crates/atuin-bench-support"` to the `[workspace] members` list. - -**Step 2: Create `Cargo.toml`** - -```toml -[package] -name = "atuin-bench-support" -edition = "2024" -description = "In-memory server backend for benchmarking atuin sync" -publish = false - -rust-version = { workspace = true } -version = { workspace = true } - -[dependencies] -atuin-server-database = { workspace = true } -atuin-server = { workspace = true } -atuin-client = { path = "../atuin-client", version = "18.16.1", features = ["sync"] } -atuin-common = { workspace = true } -async-trait = { workspace = true } -tokio = { workspace = true, features = ["net", "sync", "time", "rt"] } -tracing = { workspace = true } -uuid = { workspace = true } -time = { workspace = true } -eyre = { workspace = true } -rand = { workspace = true } -``` - -**Step 3: Implement `InMemoryDb`** - -Create `crates/atuin-bench-support/src/lib.rs` with: - -```rust -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -use async_trait::async_trait; -use atuin_common::record::{EncryptedData, HostId, Record, RecordIdx, RecordStatus}; -use atuin_server_database::models::{History, NewHistory, NewSession, NewUser, Session, User}; -use atuin_server_database::{calendar::TimePeriod, DbError, DbResult, DbSettings, Database}; -use time::{OffsetDateTime, UtcOffset}; - -#[derive(Default)] -struct Inner { - users: Vec, - sessions: Vec, - records: Vec>, - next_id: i64, -} - -#[derive(Clone, Default)] -pub struct InMemoryDb { - inner: Arc>, -} - -#[async_trait] -impl Database for InMemoryDb { - async fn new(_settings: &DbSettings) -> DbResult { - Ok(Self::default()) - } - - async fn get_session(&self, token: &str) -> DbResult { - let inner = self.inner.read().unwrap(); - inner - .sessions - .iter() - .find(|s| s.token == token) - .map(|s| Session { - id: s.id, - user_id: s.user_id, - token: s.token.clone(), - }) - .ok_or(DbError::NotFound) - } - - async fn get_session_user(&self, token: &str) -> DbResult { - let session = self.get_session(token).await?; - let inner = self.inner.read().unwrap(); - inner - .users - .iter() - .find(|u| u.id == session.user_id) - .map(|u| User { - id: u.id, - username: u.username.clone(), - email: u.email.clone(), - password: u.password.clone(), - }) - .ok_or(DbError::NotFound) - } - - async fn add_session(&self, session: &NewSession) -> DbResult<()> { - let mut inner = self.inner.write().unwrap(); - let id = inner.next_id; - inner.next_id += 1; - inner.sessions.push(Session { - id, - user_id: session.user_id, - token: session.token.clone(), - }); - Ok(()) - } - - async fn get_user(&self, username: &str) -> DbResult { - let inner = self.inner.read().unwrap(); - inner - .users - .iter() - .find(|u| u.username == username) - .map(|u| User { - id: u.id, - username: u.username.clone(), - email: u.email.clone(), - password: u.password.clone(), - }) - .ok_or(DbError::NotFound) - } - - async fn get_user_session(&self, u: &User) -> DbResult { - let inner = self.inner.read().unwrap(); - inner - .sessions - .iter() - .find(|s| s.user_id == u.id) - .map(|s| Session { - id: s.id, - user_id: s.user_id, - token: s.token.clone(), - }) - .ok_or(DbError::NotFound) - } - - async fn add_user(&self, user: &NewUser) -> DbResult { - let mut inner = self.inner.write().unwrap(); - let id = inner.next_id; - inner.next_id += 1; - inner.users.push(User { - id, - username: user.username.clone(), - email: user.email.clone(), - password: user.password.clone(), - }); - Ok(id) - } - - async fn update_user_password(&self, _u: &User) -> DbResult<()> { - Err(DbError::NotFound) - } - - async fn add_records(&self, _user: &User, records: &[Record]) -> DbResult<()> { - let mut inner = self.inner.write().unwrap(); - inner.records.extend(records.iter().cloned()); - Ok(()) - } - - async fn next_records( - &self, - _user: &User, - host: HostId, - tag: String, - start: Option, - count: u64, - ) -> DbResult>> { - let inner = self.inner.read().unwrap(); - let start = start.unwrap_or(0); - let results: Vec<_> = inner - .records - .iter() - .filter(|r| r.host == host && r.tag == tag && r.idx >= start) - .take(count as usize) - .cloned() - .collect(); - Ok(results) - } - - async fn status(&self, _user: &User) -> DbResult { - let inner = self.inner.read().unwrap(); - let mut status = RecordStatus::new(); - for record in &inner.records { - let current = status.hosts.entry(record.host).or_default(); - let tag_idx = current.entry(record.tag.clone()).or_insert(0); - if record.idx >= *tag_idx { - *tag_idx = record.idx + 1; - } - } - Ok(status) - } - - // --- Stubs for methods not needed by record sync --- - - async fn count_history(&self, _user: &User) -> DbResult { - Ok(0) - } - async fn count_history_cached(&self, _user: &User) -> DbResult { - Ok(0) - } - async fn delete_user(&self, _u: &User) -> DbResult<()> { - Err(DbError::NotFound) - } - async fn delete_history(&self, _user: &User, _id: String) -> DbResult<()> { - Err(DbError::NotFound) - } - async fn deleted_history(&self, _user: &User) -> DbResult> { - Ok(vec![]) - } - async fn delete_store(&self, _user: &User) -> DbResult<()> { - Ok(()) - } - async fn count_history_range( - &self, - _user: &User, - _range: std::ops::Range, - ) -> DbResult { - Ok(0) - } - async fn list_history( - &self, - _user: &User, - _created_after: OffsetDateTime, - _since: OffsetDateTime, - _host: &str, - _page_size: i64, - ) -> DbResult> { - Ok(vec![]) - } - async fn add_history(&self, _history: &[NewHistory]) -> DbResult<()> { - Ok(()) - } - async fn oldest_history(&self, _user: &User) -> DbResult { - Err(DbError::NotFound) - } -} -``` - -**Step 4: Verify it compiles** - -Run: `cargo check -p atuin-bench-support` -Expected: Clean compilation. - -**Step 5: Commit** - -```bash -git add crates/atuin-bench-support/ Cargo.toml -git commit -m "bench: add in-memory Database impl for sync benchmarks - -Implements the atuin-server-database Database trait backed by -Arc> storage. No Postgres needed — exercises real -HTTP round-trips through axum without external dependencies." -``` - ---- - -### Task 2: Add bench server helper to `atuin-bench-support` - -Add a `pub async fn start_bench_server()` that starts an in-process axum server with `InMemoryDb`, registers a test user, and returns a `Client` ready for benchmarking. - -**Files:** -- Modify: `crates/atuin-bench-support/src/lib.rs` - -**Step 1: Add the helper function** - -Append to `crates/atuin-bench-support/src/lib.rs`: - -```rust -use atuin_client::api_client::{self, AuthToken, Client}; -use atuin_server::{Settings as ServerSettings, launch_with_tcp_listener}; -use tokio::net::TcpListener; -use tokio::sync::oneshot; - -pub struct BenchServer { - pub addr: String, - _shutdown: oneshot::Sender<()>, - _handle: tokio::task::JoinHandle<()>, -} - -impl BenchServer { - pub async fn start() -> Self { - let settings = ServerSettings { - host: "127.0.0.1".to_owned(), - port: 0, - path: String::new(), - sync_v1_enabled: false, - open_registration: true, - max_history_length: 8192, - max_record_size: 1024 * 1024, - page_size: 1100, - register_webhook_url: None, - register_webhook_username: String::new(), - db_settings: DbSettings { - db_uri: "memory".to_owned(), - read_db_uri: None, - }, - metrics: atuin_server::settings::Metrics::default(), - fake_version: None, - }; - - let (shutdown_tx, shutdown_rx) = oneshot::channel(); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = format!("http://{}", listener.local_addr().unwrap()); - - let handle = tokio::spawn(async move { - launch_with_tcp_listener::( - settings, - listener, - shutdown_rx.unwrap_or_else(|_| ()), - ) - .await - .unwrap(); - }); - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - Self { - addr, - _shutdown: shutdown_tx, - _handle: handle, - } - } - - pub async fn register_client(&self) -> Client<'_> { - let username = uuid::Uuid::new_v4().to_string(); - let email = format!("{username}@bench.test"); - let password = "bench-password"; - - let resp = api_client::register(&self.addr, &username, &email, password) - .await - .unwrap(); - - Client::new(&self.addr, AuthToken::Token(resp.session), 5, 30).unwrap() - } -} -``` - -**Step 2: Verify it compiles** - -Run: `cargo check -p atuin-bench-support` -Expected: Clean compilation. - -**Step 3: Commit** - -```bash -git add crates/atuin-bench-support/src/lib.rs -git commit -m "bench: add BenchServer helper for in-process sync benchmarks - -Starts an axum server on a random port with InMemoryDb, registers -a test user, and returns a ready-to-use Client. No external -services required." -``` - ---- - -### Task 3: Write the sync download benchmark - -The benchmark measures what the page size change actually optimizes: the wall-clock cost of downloading N records through HTTP at different page sizes. With page_size=100, downloading 1000 records takes 10 HTTP round-trips. With page_size=1000, it takes 1. - -**Files:** -- Create: `crates/atuin/benches/sync.rs` -- Modify: `crates/atuin/Cargo.toml` (add divan + bench target) - -**Step 1: Add dev-dependencies and bench target to `crates/atuin/Cargo.toml`** - -Add to `[dev-dependencies]`: -```toml -divan = "0.1.14" -atuin-bench-support = { path = "../atuin-bench-support" } -``` - -Add at end of file: -```toml -[[bench]] -name = "sync" -harness = false -``` - -**Step 2: Write the benchmark** - -Create `crates/atuin/benches/sync.rs`: - -```rust -use atuin_bench_support::BenchServer; -use atuin_common::record::{EncryptedData, HostId, Record}; -use atuin_common::utils::uuid_v7; - -fn main() { - divan::main(); -} - -fn generate_records(n: usize) -> (HostId, String, Vec>) { - let host = HostId(uuid_v7()); - let tag = "bench".to_string(); - - let records: Vec<_> = (0..n) - .map(|i| { - Record::builder() - .host(host) - .version("v0".to_string()) - .tag(tag.clone()) - .idx(i as u64) - .data(EncryptedData { - data: format!("bench-data-{i}"), - content_encryption_key: "bench-key".to_string(), - }) - .build() - }) - .collect(); - - (host, tag, records) -} - -/// Downloads 1000 pre-uploaded records at varying page sizes. -/// -/// This directly measures the HTTP round-trip overhead that the page size -/// constant controls. At page_size=100, 1000 records require 10 HTTP requests. -/// At page_size=1000, they require 1. -#[divan::bench(args = [100, 1000], min_time = 5, sample_count = 50)] -fn download_records(bencher: divan::Bencher, page_size: u64) { - let rt = tokio::runtime::Runtime::new().unwrap(); - - let server = rt.block_on(BenchServer::start()); - let client = rt.block_on(server.register_client()); - let (host, tag, records) = generate_records(1000); - rt.block_on(client.post_records(&records)).unwrap(); - - bencher.bench(|| { - rt.block_on(async { - let mut offset = 0u64; - loop { - let page = client - .next_records(host, tag.clone(), offset, page_size) - .await - .unwrap(); - if page.is_empty() { - break; - } - offset += page.len() as u64; - } - }) - }); -} -``` - -**Step 3: Verify it compiles** - -Run: `cargo bench -p atuin --bench sync -- --test` -Expected: Compiles and lists `download_records` with both args. - -**Step 4: Run benchmark** - -Run: `cargo bench -p atuin --bench sync` -Expected: Output showing `download_records/100` significantly slower than `download_records/1000` due to 10× more HTTP round-trips. - -**Step 5: Commit** - -```bash -git add crates/atuin/benches/sync.rs crates/atuin/Cargo.toml -git commit -m "bench: add sync download benchmark comparing page sizes - -Exercises real HTTP round-trips through an in-process axum server -with InMemoryDb. Downloads 1000 records at page_size 100 vs 1000, -directly measuring the round-trip overhead the page size controls." -``` diff --git a/docs/plans/2026-07-18-commonize-rmp-msgpack-helpers.md b/docs/plans/2026-07-18-commonize-rmp-msgpack-helpers.md deleted file mode 100644 index 37c585349b4..00000000000 --- a/docs/plans/2026-07-18-commonize-rmp-msgpack-helpers.md +++ /dev/null @@ -1,1014 +0,0 @@ -# Commonize MessagePack (`rmp`) Helpers into `atuin-common` Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Replace `atuin-client`'s private `utils/rmp.rs` with a rewritten, extension-trait-based `atuin_common::rmp` module (feature-gated behind `rmp`), give it rich rstest/proptest coverage and good docs, and migrate every caller onto it — deleting six copies of hand-rolled `error_report`/trailing-bytes/array-length boilerplate. - -**Architecture:** Two extension traits — `RmpDecodeExt<'a>` implemented for `rmp::decode::Bytes<'a>` (the one decode cursor used everywhere) and `RmpEncodeExt` blanket-implemented for every `RmpWrite` — plus two error types `DecodeError`/`EncodeError` that carry legible `Display` messages and convert into `eyre::Report`. All decode reads return `DecodeError` so callers use `?` throughout instead of `.map_err(error_report)` at every read. The module lives in `atuin-common` (a leaf crate) so `atuin-client`, `atuin-scripts`, `atuin-dotfiles`, and `atuin-kv` can all share it with no dependency cycle. - -**Tech Stack:** Rust 2024, `rmp = 0.8.14`, `derive_more = 2` (`display`, `from`, `error`), `thiserror`, `eyre`; tests use `rstest`, `proptest`, `pretty_assertions` (already `atuin-common` dev-deps). - ---- - -## Background & context (read before starting) - -**What exists today.** `crates/atuin-client/src/utils/rmp.rs` (123 lines, declared `pub(crate) mod rmp;` in `utils.rs:1`) provides `EncodeError`, `DecodeError`, `read_string`, `read_optional`, `write_optional`. Its **only** real consumer is `crates/atuin-client/src/history.rs`. - -**The duplication we are killing.** Six other files never import that module — they call raw `rmp` and each re-declare an identical local helper: -```rust -fn error_report(err: E) -> eyre::Report { eyre!("{err:?}") } -``` -Copies live in: `atuin-scripts/src/store/record.rs:52`, `atuin-client/src/history/store.rs:76`, `atuin-dotfiles/src/shell.rs:45`, `atuin-dotfiles/src/store.rs:58`, `atuin-dotfiles/src/store/var.rs:52`, `atuin-kv/src/store/record.rs:39`. (`encryption.rs` inlines the same as `.map_err(|err| eyre!("{err:?}"))`.) Alongside it, the same two idioms recur ~7× each: -- **trailing-bytes guard:** `if !bytes.is_empty() { bail!("trailing bytes ... malformed") }` -- **array-length assertion:** `let n = read_array_len(..)?; ensure!(n == N, "too many entries ...")` - -`DecodeError` already converts to `eyre::Report`, so the shared fix for `error_report` is simply "make decode reads return `DecodeError` and use `?`". `expect_array_len`/`expect_eof` absorb the other two idioms. - -**Concrete usage the interface must serve** (verified against source): -- Decode cursor is `rmp::decode::Bytes<'a>` in every file. (One kv site at `atuin-kv/src/store/record.rs:78` feeds a raw `&mut &[u8]` to `read_bool`; we normalize it to `Bytes` during migration.) -- Encode writer is `&mut Vec` everywhere. `Vec: RmpWrite` (proven: `history.rs:217` returns `Result<_, EncodeError>` — default `EncodeError` — while `?`-converting `write_optional`'s error). -- `history.rs:256-260` needs a **range** field-count check (version-dependent min/max), not exact — so it keeps its own logic via `read_array_len()`; `expect_array_len` is exact-only by design (YAGNI). -- `history.rs:274,284,290` passes `read_string` as a bare fn into `read_optional`. As a method, this becomes the closure `|b| b.read_string()`. -- `write_optional` is called with `encode::write_u64` and `encode::write_str` as the writer fn (`history.rs:233-240`). - -**Naming.** Match the existing high-quality modules (`EllipsizeExt`, `EscapeNonPrintablePosixExt`, `UrlAppendExt`): traits are `RmpDecodeExt` / `RmpEncodeExt`, `...Ext` suffix, blanket/receiver impls, doc-comment-per-public-item but no filler docs. - -**Module-vs-crate name.** Inside `atuin-common`, the module is `crate::rmp` and the crate is `rmp`. A leading `rmp::` path resolves to the extern crate (extern prelude), so `use rmp::decode` inside the module file is unambiguous. Consumers must `use atuin_common::rmp::RmpDecodeExt;` (import the *items*, never `use atuin_common::rmp;` as a whole, which would shadow the crate). - -**Staging.** Stage A (Tasks 1–8) rewrites + moves the module and migrates its one real consumer (`history.rs`), then deletes the old file — this alone satisfies "completely rewrite and move it," and the tree is green at the end of Task 8. Stage B (Tasks 9–13) is the commonization payoff: migrate the six raw-`rmp` files off their local `error_report`/idioms. Stage B is independently valuable and can be deferred, but it is why the module is worth commonizing. - -**Reference — full target module.** The complete intended contents of `crates/atuin-common/src/rmp.rs` (module code, not tests) are in **Appendix A** at the end of this document. Tasks 2–5 build it up test-first; use Appendix A as the definitive source for signatures, bounds, and doc comments. - ---- - -## Stage A — rewrite, move, and land the module - -### Task 1: Add the `rmp` feature, dependency, and empty module - -**Files:** -- Modify: `crates/atuin-common/Cargo.toml` -- Modify: `crates/atuin-common/src/lib.rs:60-63` (the `pub mod ...;` block) -- Create: `crates/atuin-common/src/rmp.rs` - -**Step 1: Add the feature and optional dependency** - -In `crates/atuin-common/Cargo.toml`, under `[features]` add (after the existing `unicode = [...]` line): -```toml -# The `rmp` module: MessagePack encode/decode extension traits and error types. -rmp = ["dep:rmp"] -``` -Under `[dependencies]` add (matching the `0.8.14` pin used by the other crates — there is no workspace `rmp` entry): -```toml -rmp = { version = "0.8.14", optional = true } -``` - -**Step 2: Create a stub module** - -`crates/atuin-common/src/rmp.rs`: -```rust -//! MessagePack encode/decode helpers built on [`rmp`]. -``` - -**Step 3: Wire it into the crate root** - -In `crates/atuin-common/src/lib.rs`, add alongside the other `pub mod` declarations: -```rust -#[cfg(feature = "rmp")] -pub mod rmp; -``` - -**Step 4: Verify it builds both ways** - -Run: `cargo build -p atuin-common --features rmp` -Expected: PASS. -Run: `cargo build -p atuin-common` -Expected: PASS (module absent, `rmp` crate not compiled). - -**Step 5: Commit** -```bash -git add crates/atuin-common/Cargo.toml crates/atuin-common/src/lib.rs crates/atuin-common/src/rmp.rs -git commit -m "feat(common): add feature-gated rmp module scaffold" -``` - ---- - -### Task 2: Port the error types (`EncodeError`, `DecodeError`) - -**Files:** -- Modify: `crates/atuin-common/src/rmp.rs` - -**Step 1: Write failing tests** - -Append to `crates/atuin-common/src/rmp.rs`: -```rust -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - use rmp::decode::bytes::Bytes; - use rstest::rstest; - - // Build a real DecodeError by asking rmp to read the wrong type. - fn type_mismatch_error<'a>() -> DecodeError<'a> { - // 0xc0 is the nil marker; reading it as a u64 is a type mismatch. - let mut bytes = Bytes::new(&[0xc0]); - rmp::decode::read_u64(&mut bytes) - .map_err(DecodeError::from) - .unwrap_err() - } - - #[test] - fn type_mismatch_exposes_marker() { - assert_eq!(type_mismatch_error().type_mismatch(), Some(rmp::Marker::Null)); - } - - #[rstest] - #[case::array_len( - DecodeError::<'static>::UnexpectedArrayLen { expected: 3, actual: 5 }, - None - )] - #[case::trailing(DecodeError::<'static>::TrailingBytes { remaining: 2 }, None)] - fn structural_variants_have_no_marker( - #[case] err: DecodeError<'static>, - #[case] expected: Option, - ) { - assert_eq!(err.type_mismatch(), expected); - } - - #[test] - fn decode_error_converts_to_eyre_with_message() { - let report: eyre::Report = - DecodeError::<'static>::TrailingBytes { remaining: 4 }.into(); - assert!(report.to_string().contains("trailing")); - } - - #[test] - fn display_messages_are_legible() { - assert_eq!( - DecodeError::<'static>::UnexpectedArrayLen { expected: 3, actual: 5 }.to_string(), - "expected a MessagePack array of length 3, found 5", - ); - } -} -``` - -**Step 2: Run to verify failure** - -Run: `cargo test -p atuin-common --features rmp rmp::tests` -Expected: FAIL to compile (`DecodeError`/`EncodeError` not defined). - -**Step 3: Add the imports and error types** - -Insert the imports block and both error types from **Appendix A** (everything from the `use` lines through the `impl ... From for eyre::Report` block) above the `#[cfg(test)]` module. - -**Step 4: Run to verify pass** - -Run: `cargo test -p atuin-common --features rmp rmp::tests` -Expected: PASS. -Run: `cargo clippy -p atuin-common --features rmp -- -D warnings` -Expected: PASS. - -**Step 5: Commit** -```bash -git add crates/atuin-common/src/rmp.rs -git commit -m "feat(common): add rmp EncodeError/DecodeError types" -``` - ---- - -### Task 3: `RmpDecodeExt` core reads — `read_with`, `read_string`, `read_optional` - -**Files:** -- Modify: `crates/atuin-common/src/rmp.rs` - -**Step 1: Write failing tests** - -Add to the `tests` module: -```rust - // Encode helpers used only by tests, to build inputs. - fn enc)>(f: F) -> Vec { - let mut v = Vec::new(); - f(&mut v); - v - } - - #[test] - fn read_string_round_trips() { - let buf = enc(|v| rmp::encode::write_str(v, "héllo 🦀").unwrap()); - let mut bytes = Bytes::new(&buf); - assert_eq!(bytes.read_string().unwrap(), "héllo 🦀"); - assert!(bytes.remaining_slice().is_empty()); - } - - #[test] - fn read_string_on_wrong_type_errors_and_consumes_marker() { - // A lone nil marker: read_string must fail *and* consume the marker so a - // following read_optional can observe end-of-input correctly. - let mut bytes = Bytes::new(&[0xc0]); - assert!(bytes.read_string().is_err()); - assert!(bytes.remaining_slice().is_empty(), "marker byte must be consumed"); - } - - #[test] - fn read_with_converts_rmp_errors() { - let buf = enc(|v| rmp::encode::write_u64(v, 42).unwrap()); - let mut bytes = Bytes::new(&buf); - assert_eq!(bytes.read_with(rmp::decode::read_u64).unwrap(), 42); - } - - #[test] - fn read_optional_some_and_none() { - let some = enc(|v| rmp::encode::write_u64(v, 7).unwrap()); - let mut b = Bytes::new(&some); - assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), Some(7)); - - let none = enc(|v| rmp::encode::write_nil(v).unwrap()); - let mut b = Bytes::new(&none); - assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), None); - assert!(b.remaining_slice().is_empty()); - } - - #[test] - fn read_optional_string_via_closure() { - let buf = enc(|v| rmp::encode::write_str(v, "x").unwrap()); - let mut b = Bytes::new(&buf); - assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), Some("x".to_string())); - } - - #[test] - fn read_optional_forwards_non_nil_errors() { - // A bool where a u64 is expected is a type mismatch that is NOT nil. - let buf = enc(|v| rmp::encode::write_bool(v, true).unwrap()); - let mut b = Bytes::new(&buf); - assert!(b.read_optional(rmp::decode::read_u64).is_err()); - } -``` - -**Step 2: Run to verify failure** - -Run: `cargo test -p atuin-common --features rmp rmp::tests` -Expected: FAIL (`RmpDecodeExt` / methods not defined). - -**Step 3: Implement the trait + these three methods** - -Add the `RmpDecodeExt<'a>` trait declaration and the `impl<'a> RmpDecodeExt<'a> for Bytes<'a>` block from **Appendix A**, but for this task include **only** `read_with`, `read_string`, and `read_optional` (leave `read_array_len`/`expect_array_len`/`expect_eof` for Task 4). Keep the trait declaration complete (all six method signatures) and add a `todo!()`-free partial impl — i.e. add the three later methods' signatures to the trait now but implement the remaining three in Task 4. To keep the crate compiling between tasks, add all six method bodies now using Appendix A (implementing all six here is fine; Task 4 just adds their tests). **Recommended:** implement all six method bodies in this task, and add the validation *tests* in Task 4. - -**Step 4: Run to verify pass** - -Run: `cargo test -p atuin-common --features rmp rmp::tests` -Expected: PASS. -Run: `cargo clippy -p atuin-common --features rmp -- -D warnings` -Expected: PASS. - -**Step 5: Commit** -```bash -git add crates/atuin-common/src/rmp.rs -git commit -m "feat(common): add RmpDecodeExt read_with/read_string/read_optional" -``` - ---- - -### Task 4: `RmpDecodeExt` validation — `read_array_len`, `expect_array_len`, `expect_eof` - -**Files:** -- Modify: `crates/atuin-common/src/rmp.rs` - -**Step 1: Write failing tests** - -Add to the `tests` module: -```rust - fn array_of(len: u32) -> Vec { - enc(|v| rmp::encode::write_array_len(v, len).unwrap()) - } - - #[test] - fn expect_array_len_exact_ok() { - let buf = array_of(3); - let mut b = Bytes::new(&buf); - assert_eq!(b.expect_array_len(3).unwrap(), 3); - } - - #[test] - fn expect_array_len_mismatch_reports_expected_and_actual() { - let buf = array_of(5); - let mut b = Bytes::new(&buf); - match b.expect_array_len(3) { - Err(DecodeError::UnexpectedArrayLen { expected, actual }) => { - assert_eq!((expected, actual), (3, 5)); - } - other => panic!("expected UnexpectedArrayLen, got {other:?}"), - } - } - - #[test] - fn read_array_len_returns_count_for_manual_range_checks() { - let buf = array_of(9); - let mut b = Bytes::new(&buf); - assert_eq!(b.read_array_len().unwrap(), 9); - } - - #[test] - fn expect_eof_ok_when_consumed() { - let buf = enc(|v| rmp::encode::write_u8(v, 1).unwrap()); - let mut b = Bytes::new(&buf); - b.read_with(rmp::decode::read_u8).unwrap(); - assert!(b.expect_eof().is_ok()); - } - - #[test] - fn expect_eof_reports_remaining() { - let b = Bytes::new(&[0x01, 0x02, 0x03]); - match b.expect_eof() { - Err(DecodeError::TrailingBytes { remaining }) => assert_eq!(remaining, 3), - other => panic!("expected TrailingBytes, got {other:?}"), - } - } -``` - -**Step 2: Run to verify failure** - -Run: `cargo test -p atuin-common --features rmp rmp::tests` -Expected: PASS if Task 3 already implemented all six bodies (recommended path); otherwise FAIL until you add the three method bodies from Appendix A. If FAIL, add them now. - -**Step 3: Ensure the three method bodies are present** - -Confirm `read_array_len`, `expect_array_len`, `expect_eof` match Appendix A. - -**Step 4: Run to verify pass** - -Run: `cargo test -p atuin-common --features rmp rmp::tests` -Expected: PASS. -Run: `cargo clippy -p atuin-common --features rmp -- -D warnings` -Expected: PASS. - -**Step 5: Commit** -```bash -git add crates/atuin-common/src/rmp.rs -git commit -m "feat(common): add RmpDecodeExt array-len and eof checks" -``` - ---- - -### Task 5: `RmpEncodeExt::write_optional` - -**Files:** -- Modify: `crates/atuin-common/src/rmp.rs` - -**Step 1: Write failing tests** - -Add to the `tests` module: -```rust - #[test] - fn write_optional_some_then_read_back() { - let mut out = Vec::new(); - out.write_optional(Some(99u64), rmp::encode::write_u64).unwrap(); - let mut b = Bytes::new(&out); - assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), Some(99)); - } - - #[test] - fn write_optional_none_writes_nil() { - let mut out = Vec::new(); - out.write_optional::(None, rmp::encode::write_u64).unwrap(); - let mut b = Bytes::new(&out); - assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), None); - } - - #[test] - fn write_optional_str() { - let mut out = Vec::new(); - out.write_optional(Some("hi"), rmp::encode::write_str).unwrap(); - let mut b = Bytes::new(&out); - assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), Some("hi".to_string())); - } -``` - -**Step 2: Run to verify failure** - -Run: `cargo test -p atuin-common --features rmp rmp::tests` -Expected: FAIL (`write_optional` not defined). - -**Step 3: Implement** - -Add the `RmpEncodeExt` trait and its blanket `impl` from **Appendix A**. - -**Step 4: Run to verify pass** - -Run: `cargo test -p atuin-common --features rmp rmp::tests` -Expected: PASS. -Run: `cargo clippy -p atuin-common --features rmp -- -D warnings` -Expected: PASS. - -**Step 5: Commit** -```bash -git add crates/atuin-common/src/rmp.rs -git commit -m "feat(common): add RmpEncodeExt write_optional" -``` - ---- - -### Task 6: Property tests (round-trips + never-panics) - -**Files:** -- Modify: `crates/atuin-common/src/rmp.rs` - -**Step 1: Write the proptests** - -Add to the `tests` module: -```rust - use proptest::prelude::*; - - proptest! { - #![proptest_config(ProptestConfig::with_cases(1024))] - - // A string survives an encode -> read_string round trip exactly. - #[test] - fn string_round_trips(s in r"(?s).*") { - let buf = enc(|v| rmp::encode::write_str(v, &s).unwrap()); - let mut b = Bytes::new(&buf); - prop_assert_eq!(b.read_string().unwrap(), s); - prop_assert!(b.remaining_slice().is_empty()); - } - - // Option survives write_optional -> read_optional. - #[test] - fn optional_u64_round_trips(v in proptest::option::of(any::())) { - let mut out = Vec::new(); - out.write_optional(v, rmp::encode::write_u64).unwrap(); - let mut b = Bytes::new(&out); - prop_assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), v); - } - - // Option survives the closure-based optional round trip. - #[test] - fn optional_string_round_trips(v in proptest::option::of(r"(?s).*")) { - let mut out = Vec::new(); - out.write_optional(v.as_deref(), rmp::encode::write_str).unwrap(); - let mut b = Bytes::new(&out); - prop_assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), v); - } - - // A full array record (len + fields + optional tail) round trips, and the - // cursor is exactly exhausted afterwards. - #[test] - fn record_round_trips( - id in r"[a-z]{0,16}", - ts in any::(), - deleted in proptest::option::of(any::()), - ) { - let mut out = Vec::new(); - rmp::encode::write_array_len(&mut out, 3).unwrap(); - rmp::encode::write_str(&mut out, &id).unwrap(); - rmp::encode::write_u64(&mut out, ts).unwrap(); - out.write_optional(deleted, rmp::encode::write_u64).unwrap(); - - let mut b = Bytes::new(&out); - prop_assert_eq!(b.expect_array_len(3).unwrap(), 3); - prop_assert_eq!(b.read_string().unwrap(), id); - prop_assert_eq!(b.read_with(rmp::decode::read_u64).unwrap(), ts); - prop_assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), deleted); - prop_assert!(b.expect_eof().is_ok()); - } - - // Reads never panic on arbitrary bytes — they return Err instead. - #[test] - fn reads_never_panic(raw in proptest::collection::vec(any::(), 0..64)) { - let mut b = Bytes::new(&raw); - let _ = b.read_string(); - let mut b = Bytes::new(&raw); - let _ = b.read_array_len(); - let mut b = Bytes::new(&raw); - let _ = b.read_optional(rmp::decode::read_u64); - } - } -``` - -**Step 2: Run** - -Run: `cargo test -p atuin-common --features rmp rmp::tests` -Expected: PASS. - -**Step 3: Commit** -```bash -git add crates/atuin-common/src/rmp.rs -git commit -m "test(common): property tests for rmp round-trips and panic-safety" -``` - ---- - -### Task 7: Migrate `history.rs` onto `atuin_common::rmp` - -**Files:** -- Modify: `crates/atuin-client/Cargo.toml` (enable the `rmp` feature on `atuin-common`) -- Modify: `crates/atuin-client/src/history.rs:1-13` (imports) and `:217-315` (serialize/deserialize) - -**Step 1: Enable the feature** - -In `crates/atuin-client/Cargo.toml`, under `[dependencies.atuin-common]`, add a features line: -```toml -[dependencies.atuin-common] -path = "../atuin-common" -features = ["rmp"] -``` -(Leave the `[dev-dependencies.atuin-common]` block as-is; cargo unifies features.) - -**Step 2: Swap the import** - -In `crates/atuin-client/src/history.rs`, replace: -```rust -use crate::utils::rmp::{DecodeError, EncodeError, read_optional, read_string, write_optional}; -``` -with: -```rust -use atuin_common::rmp::{DecodeError, EncodeError, RmpDecodeExt as _, RmpEncodeExt as _}; -``` -Keep the existing `use rmp::decode::{self, Bytes};` and `use rmp::encode;` lines — the raw crate is still used directly. - -**Step 3: Rewrite `serialize` (lines ~217-242)** - -Change the three `write_optional(&mut output, VALUE, WRITER)?` free-function calls to method calls: -```rust - output.write_optional( - self.deleted_at.map(|d| d.unix_timestamp_nanos() as u64), - encode::write_u64, - )?; - encode::write_str(&mut output, self.author.as_str())?; - output.write_optional(self.intent.as_deref(), encode::write_str)?; - output.write_optional(self.shell.as_deref(), encode::write_str)?; -``` - -**Step 4: Rewrite `deserialize` (lines ~244-297)** - -Replace free-function/`map_err(DecodeError::from)` calls with method calls. Note `read_string` becomes a method, so where it was passed as a bare fn to `read_optional`, use a closure: -```rust - let real_version = bytes.read_with(decode::read_u16)?; - // ... unchanged range check on real_version ... - - let nfields = bytes.read_array_len()?; - // ... unchanged version-dependent min/max range check + bail! ... - - let id = bytes.read_string()?; - let timestamp = bytes.read_with(decode::read_u64)?; - let duration = bytes.read_with(decode::read_int)?; - let exit = bytes.read_with(decode::read_int)?; - - let command = bytes.read_string()?; - let cwd = bytes.read_string()?; - let session = bytes.read_string()?; - let hostname = bytes.read_string()?; - let deleted_at = bytes.read_optional(decode::read_u64)?; - - let author = if version >= Version::One { - bytes.read_optional(|b| b.read_string())? - } else { - None - }; - // ... intent: same closure form ... - // ... shell: same closure form ... - - if version < Version::Two && !bytes.remaining_slice().is_empty() { - bail!("trailing bytes in encoded history. malformed") - } -``` -Keep the version-dependent trailing-bytes check as-is (it is conditional, not a plain `expect_eof`). - -**Step 5: Verify** - -Run: `cargo test -p atuin-client --lib history` -Expected: PASS (existing history serialize/deserialize round-trip tests still pass). -Run: `cargo clippy -p atuin-client -- -D warnings` -Expected: PASS. - -**Step 6: Commit** -```bash -git add crates/atuin-client/Cargo.toml crates/atuin-client/src/history.rs -git commit -m "refactor(client): use atuin_common::rmp in history serde" -``` - ---- - -### Task 8: Delete the old module - -**Files:** -- Delete: `crates/atuin-client/src/utils/rmp.rs` -- Modify: `crates/atuin-client/src/utils.rs:1` (remove `pub(crate) mod rmp;`) - -**Step 1: Confirm no remaining references** - -Run: `rg 'utils::rmp|utils/rmp' crates/` -Expected: no matches. - -**Step 2: Delete + unwire** -```bash -git rm crates/atuin-client/src/utils/rmp.rs -``` -Remove the `pub(crate) mod rmp;` line from `crates/atuin-client/src/utils.rs`. If that leaves `utils.rs` empty or `utils` unused, check `rg 'mod utils|utils::' crates/atuin-client/src` and clean up only if genuinely dead (do not remove other utils). - -**Step 3: Verify** - -Run: `cargo build -p atuin-client` -Expected: PASS. -Run: `cargo test -p atuin-client` -Expected: PASS. - -**Step 4: Commit** -```bash -git add -A -git commit -m "refactor(client): delete now-unused utils/rmp module" -``` - -**Stage A is complete here. The tree is green and the module is moved.** - ---- - -## Stage B — commonization payoff: migrate the raw-`rmp` callers - -Each Stage B task follows the same shape: enable `features = ["rmp"]` on the crate's `atuin-common` dependency (skip if already set), delete the local `error_report` fn, and replace the idioms: - -| Old idiom | New | -|---|---| -| `decode::read_x(&mut b).map_err(error_report)?` | `b.read_with(decode::read_x)?` | -| `let (s, rest) = read_str_from_slice(..).map_err(error_report)?; s.to_owned()` | `b.read_string()?` (owned) | -| `let n = read_array_len(..)?; ensure!(n == N, "...")` | `b.expect_array_len(N)?` | -| `if !bytes.is_empty() { bail!("trailing ...") }` | `b.expect_eof()?` (optionally `.wrap_err("")`) | - -Import `use atuin_common::rmp::{RmpDecodeExt as _, RmpEncodeExt as _};` (add `EncodeError`/`DecodeError` only where a type is named). Keep each file's `use rmp::{decode, encode};`. Normalize any `&mut &[u8]` reader to a `Bytes` cursor so the trait applies. - -> Behavior note: `expect_eof`/`expect_array_len` produce generic-but-informative messages (they include the actual counts/bytes). This replaces record-specific strings like "too many entries in v0 kv record". Where the specific context matters for debugging, append `.wrap_err("v0 kv record")?`. Confirm each crate's existing (de)serialize round-trip tests still pass after migration — the wire format must not change. - -### Task 9: `atuin-client` — `encryption.rs` and `history/store.rs` - -**Files:** -- Modify: `crates/atuin-client/src/encryption.rs:71-107` (`decode_key`) — replace the inline `.map_err(|err| eyre!("{err:?}"))` closures with `b.read_with(...)`; replace `ensure!(len == 32, ...)` on `read_array_len` with `b.expect_array_len(32)?` (the `Bin8`/`read_bin_len` branch keeps its own `ensure!` since `read_bin_len` isn't an array). Feature already enabled in Task 7. -- Modify: `crates/atuin-client/src/history/store.rs:73-114` (`deserialize`) — delete `error_report` (lines 76-78); use `b.read_with(decode::read_u8)`, `b.read_with(decode::read_bin_len)`; in the `Delete` arm replace `read_str_from_slice(..).map_err(error_report)?` + trailing check with `let id = bytes.read_string()?; bytes.expect_eof()?;`. - -**Step 1-2:** Make the edits above; keep the `Create` arm handing `bytes.remaining_slice()` to `History::deserialize` unchanged. - -**Step 3: Verify** - -Run: `cargo test -p atuin-client` -Expected: PASS. -Run: `cargo clippy -p atuin-client -- -D warnings` -Expected: PASS. - -**Step 4: Commit** -```bash -git add crates/atuin-client/src/encryption.rs crates/atuin-client/src/history/store.rs -git commit -m "refactor(client): use atuin_common::rmp in encryption and history store" -``` - ---- - -### Task 10: `atuin-scripts` — `store/record.rs` and `store/script.rs` - -**Files:** -- Modify: `crates/atuin-scripts/Cargo.toml` — add `features = ["rmp"]` to the `atuin-common` dependency. -- Modify: `crates/atuin-scripts/src/store/record.rs:49-91` — delete `error_report`; use `bytes.read_with(decode::read_u8)`, `bytes.read_with(decode::read_bin_len)`; in the delete arm, `bytes.read_string()?` instead of `read_str_from_slice`. -- Modify: `crates/atuin-scripts/src/store/script.rs:65-104` — this file uses `.unwrap()` throughout; convert to `?` via the trait: `bytes.expect_array_len(6)?`, `bytes.read_string()?` (×4 + the tags loop + script), and the tags nested `read_array_len` via `bytes.read_array_len()?`; replace the trailing `if !bytes.is_empty() { bail! }` with `bytes.expect_eof()?`. - -> `script.rs` currently `.unwrap()`s — migrating to `?` is a strict improvement (malformed data becomes an error instead of a panic). Keep the `ensure!(nfields == 6, ...)` semantics via `expect_array_len(6)`. - -**Step 1-2:** Make the edits. Normalize the alternating `Bytes`/`&[u8]` handling to a single `Bytes` cursor throughout (`read_string` advances it), removing the `Bytes::new(rest)` re-wrapping. - -**Step 3: Verify** - -Run: `cargo test -p atuin-scripts` -Expected: PASS. -Run: `cargo clippy -p atuin-scripts -- -D warnings` -Expected: PASS. - -**Step 4: Commit** -```bash -git add crates/atuin-scripts/Cargo.toml crates/atuin-scripts/src/store/record.rs crates/atuin-scripts/src/store/script.rs -git commit -m "refactor(scripts): use atuin_common::rmp in record/script serde" -``` - ---- - -### Task 11: `atuin-dotfiles` — `shell.rs`, `store.rs`, `store/var.rs` - -**Files:** -- Modify: `crates/atuin-dotfiles/Cargo.toml` — add `features = ["rmp"]` to `atuin-common`. -- Modify: `crates/atuin-dotfiles/src/shell.rs:44-76` (`Var::deserialize(bytes: &mut decode::Bytes)`) — delete `error_report`; `bytes.expect_array_len(3)?`; `bytes.read_string()?` for name/value; `bytes.read_with(decode::read_bool)?`; `bytes.expect_eof()?`. `Var::serialize(&mut Vec)` writer stays raw `encode::*`. -- Modify: `crates/atuin-dotfiles/src/store.rs:55-123` — delete `error_report`; `read_u8` via `read_with`; `expect_array_len(2)`/`expect_array_len(1)`; `read_string()`; `expect_eof()`. -- Modify: `crates/atuin-dotfiles/src/store/var.rs:49-99` — delete `error_report`; `read_u8` via `read_with`; delete-arm `expect_array_len(1)` + `read_string()` + `expect_eof()`; the create arm still delegates to `Var::deserialize(&mut bytes)`. - -**Step 1-2:** Make the edits. `shell.rs`'s `&mut decode::Bytes` receiver is exactly what `RmpDecodeExt for Bytes` needs — no signature change. - -**Step 3: Verify** - -Run: `cargo test -p atuin-dotfiles` -Expected: PASS. -Run: `cargo clippy -p atuin-dotfiles -- -D warnings` -Expected: PASS. - -**Step 4: Commit** -```bash -git add crates/atuin-dotfiles/Cargo.toml crates/atuin-dotfiles/src/shell.rs crates/atuin-dotfiles/src/store.rs crates/atuin-dotfiles/src/store/var.rs -git commit -m "refactor(dotfiles): use atuin_common::rmp across var/alias serde" -``` - ---- - -### Task 12: `atuin-kv` — `store/record.rs` - -**Files:** -- Modify: `crates/atuin-kv/Cargo.toml` — add `features = ["rmp"]` to `atuin-common`. -- Modify: `crates/atuin-kv/src/store/record.rs:36-102` — delete `error_report`; both version arms use `expect_array_len(3)` / `expect_array_len(4)`, `read_string()`, `expect_eof()`. Critically, normalize the `v1` branch's `read_bool` on a raw `&mut &[u8]` (line 78) to operate on the `Bytes` cursor: `let has_value = bytes.read_with(decode::read_bool)?;` then continue reading from the same `bytes` cursor (`if has_value { Some(bytes.read_string()?) } else { None }`), and `bytes.expect_eof()?`. - -**Step 1-2:** Make the edits. This removes the only `&mut &[u8]` reader in the codebase, unifying on `Bytes`. - -**Step 3: Verify** - -Run: `cargo test -p atuin-kv` -Expected: PASS. -Run: `cargo clippy -p atuin-kv -- -D warnings` -Expected: PASS. - -**Step 4: Commit** -```bash -git add crates/atuin-kv/Cargo.toml crates/atuin-kv/src/store/record.rs -git commit -m "refactor(kv): use atuin_common::rmp in record serde" -``` - ---- - -### Task 13: Full-workspace verification - -**Step 1: Confirm the boilerplate is gone** - -Run: `rg 'fn error_report' crates/` -Expected: no matches. -Run: `rg 'map_err\(\|err\| eyre!\("\{err:\?\}"\)\)' crates/` -Expected: no matches. - -**Step 2: Build, test, lint the whole workspace** - -Run: `cargo build --workspace` -Expected: PASS. -Run: `cargo test --workspace` -Expected: PASS. -Run: `cargo clippy --workspace --all-targets -- -D warnings` -Expected: PASS. - -**Step 3: Verify the feature-off path still compiles** - -Run: `cargo build -p atuin-common` (no `rmp` feature) -Expected: PASS (module excluded). - -**Step 4: Commit any final touch-ups** -```bash -git add -A -git commit -m "chore: verify workspace after rmp commonization" -``` - ---- - -## Appendix A — Complete target `crates/atuin-common/src/rmp.rs` (module code, no tests) - -```rust -//! MessagePack encode/decode helpers built on [`rmp`]. -//! -//! `rmp`'s own error types are awkward: some don't implement [`Display`] for -//! every `E`, none say which variant they are, and the decode cursor offers no -//! owned-string read, no nil-aware optional read, and no structural checks -//! (array length, end-of-input). This module fills those gaps with two -//! extension traits — [`RmpDecodeExt`] for reading a [`Bytes`] cursor and -//! [`RmpEncodeExt`] for writing — plus [`DecodeError`] / [`EncodeError`], which -//! carry legible messages and convert cleanly into [`eyre::Report`]. -//! -//! Because every decode read returns [`DecodeError`], a decoder can use `?` -//! throughout and convert once at the boundary, rather than hand-rolling a -//! `map_err` at every read. -//! -//! [`Display`]: std::fmt::Display - -use rmp::Marker; -use rmp::decode::bytes::{Bytes, BytesReadError}; -use rmp::decode::{self, DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError}; -use rmp::encode::{self, RmpWrite, RmpWriteErr, ValueWriteError}; - -/// An error encountered while encoding a MessagePack value. -/// -/// Wraps [`ValueWriteError`] with a message that names the failing write and its -/// inner I/O error — neither of which `rmp`'s own [`Display`] reports. Implements -/// [`std::error::Error`], so it converts into [`eyre::Report`] with `?`. -/// -/// [`Display`]: std::fmt::Display -#[derive(Debug, derive_more::Display, derive_more::From, thiserror::Error)] -#[display("could not write MessagePack value: {_0:?}")] -pub struct EncodeError(ValueWriteError); - -/// An error encountered while decoding a MessagePack value. -/// -/// Wraps the three error types `rmp`'s decode functions return, and adds two -/// structural variants for checks `rmp` does not perform itself: -/// [`UnexpectedArrayLen`](Self::UnexpectedArrayLen) and -/// [`TrailingBytes`](Self::TrailingBytes). -/// -/// Converts into [`eyre::Report`] via a manual `From` rather than a -/// [`std::error::Error`] impl, because a [`DecodeError`] is not, in general, -/// `'static`. -/// -/// [`Display`]: std::fmt::Display -#[derive(Debug, derive_more::Display)] -pub enum DecodeError<'a, E: RmpReadErr = BytesReadError> { - /// The next value was not a valid UTF-8 string. - #[display("could not decode MessagePack string: {_0:?}")] - DecodeString(DecodeStringError<'a, E>), - /// The next value was not the expected number. - #[display("could not decode MessagePack number: {_0:?}")] - NumValueRead(NumValueReadError), - /// The next value could not be decoded. - #[display("could not decode MessagePack value: {_0:?}")] - ValueRead(ValueReadError), - /// An array marker did not have the expected length. - #[display("expected a MessagePack array of length {expected}, found {actual}")] - UnexpectedArrayLen { expected: u32, actual: u32 }, - /// Input remained after a value was fully decoded. - #[display("{remaining} trailing byte(s) after decoding MessagePack value")] - TrailingBytes { remaining: usize }, -} - -impl<'a, E: RmpReadErr> From> for DecodeError<'a, E> { - fn from(e: DecodeStringError<'a, E>) -> Self { - Self::DecodeString(e) - } -} - -impl From> for DecodeError<'_, E> { - fn from(e: NumValueReadError) -> Self { - Self::NumValueRead(e) - } -} - -impl From> for DecodeError<'_, E> { - fn from(e: ValueReadError) -> Self { - Self::ValueRead(e) - } -} - -impl DecodeError<'_, E> { - /// If this is a type mismatch, the [`Marker`] found instead of the expected - /// type. [`RmpDecodeExt::read_optional`] uses this to recognise nil. - pub fn type_mismatch(&self) -> Option { - match self { - Self::DecodeString(DecodeStringError::TypeMismatch(m)) - | Self::NumValueRead(NumValueReadError::TypeMismatch(m)) - | Self::ValueRead(ValueReadError::TypeMismatch(m)) => Some(*m), - _ => None, - } - } -} - -impl From> for eyre::Report { - fn from(e: DecodeError<'_, E>) -> Self { - eyre::eyre!("{e}") - } -} - -/// Reading helpers for a MessagePack [`Bytes`] cursor. -/// -/// Every method reports failures as [`DecodeError`], so decoders use `?` -/// throughout and convert once at the boundary. -pub trait RmpDecodeExt<'a> { - /// Run an `rmp` decode function, converting its error into [`DecodeError`]. - /// - /// Lets raw `rmp::decode` functions compose with `?`: - /// `bytes.read_with(rmp::decode::read_u64)?`. - fn read_with(&mut self, read: F) -> Result> - where - F: FnOnce(&mut Self) -> Result, - E: Into>; - - /// Read an owned [`String`]. - fn read_string(&mut self) -> Result>; - - /// Read a value that may be encoded as nil, returning [`None`] for nil. - /// - /// `read` decodes a `T`; if it fails specifically because it found - /// [`Marker::Null`], this yields [`None`] with the cursor left just past the - /// nil. Any other error is forwarded unchanged. - fn read_optional(&mut self, read: F) -> Result, DecodeError<'a>> - where - F: FnOnce(&mut Self) -> Result, - E: Into>; - - /// Read an array-length marker. - fn read_array_len(&mut self) -> Result>; - - /// Read an array-length marker and require it to equal `expected`. - /// - /// Returns [`DecodeError::UnexpectedArrayLen`] otherwise. For a - /// forward-compatible field count, use [`read_array_len`](Self::read_array_len) - /// and range-check the value yourself. - fn expect_array_len(&mut self, expected: u32) -> Result>; - - /// Succeed only if the cursor is at end-of-input, else - /// [`DecodeError::TrailingBytes`] — the standard malformed-record guard after - /// a fixed set of fields. - fn expect_eof(&self) -> Result<(), DecodeError<'a>>; -} - -impl<'a> RmpDecodeExt<'a> for Bytes<'a> { - fn read_with(&mut self, read: F) -> Result> - where - F: FnOnce(&mut Self) -> Result, - E: Into>, - { - read(self).map_err(Into::into) - } - - fn read_string(&mut self) -> Result> { - let slice = self.remaining_slice(); - let (string, rest) = match decode::read_str_from_slice(slice) { - Ok(pair) => pair, - Err(e) => { - if let DecodeStringError::TypeMismatch(_) = e { - // rmp's decode functions consume the marker byte on a type - // mismatch; do the same so `read_optional` can detect nil. - self.read_u8() - .expect("TypeMismatch implies the stream contains a marker byte"); - } - return Err(e.into()); - } - }; - *self = Bytes::new(rest); - Ok(string.into()) - } - - fn read_optional(&mut self, read: F) -> Result, DecodeError<'a>> - where - F: FnOnce(&mut Self) -> Result, - E: Into>, - { - match read(self) { - Ok(v) => Ok(Some(v)), - Err(e) => { - let e = e.into(); - if let Some(Marker::Null) = e.type_mismatch() { - Ok(None) - } else { - Err(e) - } - } - } - } - - fn read_array_len(&mut self) -> Result> { - decode::read_array_len(self).map_err(Into::into) - } - - fn expect_array_len(&mut self, expected: u32) -> Result> { - let actual = self.read_array_len()?; - if actual == expected { - Ok(actual) - } else { - Err(DecodeError::UnexpectedArrayLen { expected, actual }) - } - } - - fn expect_eof(&self) -> Result<(), DecodeError<'a>> { - let remaining = self.remaining_slice().len(); - if remaining == 0 { - Ok(()) - } else { - Err(DecodeError::TrailingBytes { remaining }) - } - } -} - -/// Writing helpers for a MessagePack output buffer. -pub trait RmpEncodeExt: RmpWrite { - /// Write an optional value, encoding [`None`] as [`Marker::Null`]. - /// - /// The mirror of [`RmpDecodeExt::read_optional`]: `write` encodes the inner - /// value when `value` is [`Some`]. - fn write_optional( - &mut self, - value: Option, - write: F, - ) -> Result<(), EncodeError> - where - F: FnOnce(&mut Self, T) -> Result<(), ValueWriteError>; -} - -impl RmpEncodeExt for W { - fn write_optional( - &mut self, - value: Option, - write: F, - ) -> Result<(), EncodeError> - where - F: FnOnce(&mut Self, T) -> Result<(), ValueWriteError>, - { - match value { - Some(v) => write(self, v).map_err(EncodeError::from), - None => encode::write_nil(self) - .map_err(|e| EncodeError::from(ValueWriteError::InvalidMarkerWrite(e))), - } - } -} -``` - -> **If the build disagrees with Appendix A**, trust the compiler and the *old* module's proven signatures (`crates/atuin-client/src/utils/rmp.rs` in git history) over this appendix — particularly the exact `RmpWrite::Error` / `ValueWriteError` bounds. The old `read_string`/`read_optional`/`write_optional` bodies are known-good; the only substantive additions here are the two structural `DecodeError` variants and the `read_with`/`read_array_len`/`expect_array_len`/`expect_eof` methods. From fff695aaef3feca8848bac25a116d12863d0f8bd Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 23:16:49 -0700 Subject: [PATCH 17/27] refactor(common): make rmp helpers free functions instead of extension traits --- crates/atuin-client/src/encryption.rs | 8 +- crates/atuin-client/src/history.rs | 39 +-- crates/atuin-client/src/history/store.rs | 10 +- crates/atuin-common/src/rmp.rs | 307 ++++++++++------------- crates/atuin-dotfiles/src/shell.rs | 12 +- crates/atuin-dotfiles/src/store.rs | 18 +- crates/atuin-dotfiles/src/store/var.rs | 10 +- crates/atuin-kv/src/store/record.rs | 24 +- crates/atuin-scripts/src/store/record.rs | 10 +- crates/atuin-scripts/src/store/script.rs | 20 +- 10 files changed, 212 insertions(+), 246 deletions(-) diff --git a/crates/atuin-client/src/encryption.rs b/crates/atuin-client/src/encryption.rs index ea6948350c4..e8b54652f21 100644 --- a/crates/atuin-client/src/encryption.rs +++ b/crates/atuin-client/src/encryption.rs @@ -17,7 +17,7 @@ use eyre::{Context, Result, bail, ensure}; use fs_err as fs; use rmp::Marker; -use atuin_common::rmp::RmpDecodeExt as _; +use atuin_common::rmp::{expect_array_len, read_with}; use crate::settings::Settings; @@ -86,18 +86,18 @@ pub fn decode_key(key: String) -> Result { match Marker::from_u8(buf[0]) { Marker::Bin8 => { - let len = bytes.read_with(decode::read_bin_len)?; + let len = read_with(&mut bytes, decode::read_bin_len)?; ensure!(len == 32, "encryption key is not the correct size"); let key = <[u8; 32]>::try_from(bytes.remaining_slice()) .context("could not decode encryption key")?; Ok(key.into()) } Marker::Array16 => { - bytes.expect_array_len(32)?; + expect_array_len(&mut bytes, 32)?; let mut key = Key::default(); for i in &mut key { - *i = bytes.read_with(decode::read_int)?; + *i = read_with(&mut bytes, decode::read_int)?; } Ok(key) } diff --git a/crates/atuin-client/src/history.rs b/crates/atuin-client/src/history.rs index a0b5bae3e39..e9c4cbc03a4 100644 --- a/crates/atuin-client/src/history.rs +++ b/crates/atuin-client/src/history.rs @@ -10,7 +10,9 @@ use eyre::{Result, bail}; use crate::secrets::SECRET_PATTERNS_RE; use crate::settings::Settings; use crate::utils::get_host_user; -use atuin_common::rmp::{EncodeError, RmpDecodeExt as _, RmpEncodeExt as _}; +use atuin_common::rmp::{ + EncodeError, read_array_len, read_optional, read_string, read_with, write_optional, +}; use time::OffsetDateTime; pub(crate) mod builder; @@ -230,13 +232,14 @@ impl History { encode::write_str(&mut output, &self.session)?; encode::write_str(&mut output, &self.hostname)?; - output.write_optional( + write_optional( + &mut output, self.deleted_at.map(|d| d.unix_timestamp_nanos() as u64), encode::write_u64, )?; encode::write_str(&mut output, self.author.as_str())?; - output.write_optional(self.intent.as_deref(), encode::write_str)?; - output.write_optional(self.shell.as_deref(), encode::write_str)?; + write_optional(&mut output, self.intent.as_deref(), encode::write_str)?; + write_optional(&mut output, self.shell.as_deref(), encode::write_str)?; Ok(DecryptedData(output)) } @@ -247,30 +250,30 @@ impl History { let mut bytes = Bytes::new(bytes); - let real_version = bytes.read_with(decode::read_u16)?; + let real_version = read_with(&mut bytes, decode::read_u16)?; if real_version != version.as_int() { bail!("expected to decode {version} record, found v{real_version}"); } - let nfields = bytes.read_array_len()?; + let nfields = read_array_len(&mut bytes)?; let min_fields = version.min_fields(); if nfields < min_fields || version.max_fields().is_some_and(|max| nfields > max) { bail!("unexpected number of fields ({nfields}) for history version {version}"); } - let id = bytes.read_string()?; - let timestamp = bytes.read_with(decode::read_u64)?; - let duration = bytes.read_with(decode::read_int)?; - let exit = bytes.read_with(decode::read_int)?; + let id = read_string(&mut bytes)?; + let timestamp = read_with(&mut bytes, decode::read_u64)?; + let duration = read_with(&mut bytes, decode::read_int)?; + let exit = read_with(&mut bytes, decode::read_int)?; - let command = bytes.read_string()?; - let cwd = bytes.read_string()?; - let session = bytes.read_string()?; - let hostname = bytes.read_string()?; - let deleted_at = bytes.read_optional(decode::read_u64)?; + let command = read_string(&mut bytes)?; + let cwd = read_string(&mut bytes)?; + let session = read_string(&mut bytes)?; + let hostname = read_string(&mut bytes)?; + let deleted_at = read_optional(&mut bytes, decode::read_u64)?; let author = if version >= Version::One { - bytes.read_optional(|b| b.read_string())? + read_optional(&mut bytes, read_string)? } else { None }; @@ -280,13 +283,13 @@ impl History { Version::One => nfields > min_fields, Version::Two => true, } { - bytes.read_optional(|b| b.read_string())? + read_optional(&mut bytes, read_string)? } else { None }; let shell = if version >= Version::Two { - bytes.read_optional(|b| b.read_string())? + read_optional(&mut bytes, read_string)? } else { None }; diff --git a/crates/atuin-client/src/history/store.rs b/crates/atuin-client/src/history/store.rs index 94a4d09ab5b..5309c5187ce 100644 --- a/crates/atuin-client/src/history/store.rs +++ b/crates/atuin-client/src/history/store.rs @@ -10,7 +10,7 @@ use crate::{ record::{encryption::PASETO_V4, sqlite_store::SqliteStore, store::Store}, }; use atuin_common::record::{DecryptedData, Host, HostId, Record, RecordId, RecordIdx}; -use atuin_common::rmp::RmpDecodeExt as _; +use atuin_common::rmp::{expect_eof, read_string, read_with}; use super::{HISTORY_TAG, History, HistoryId, Version}; @@ -76,14 +76,14 @@ impl HistoryRecord { let mut bytes = Bytes::new(&bytes.0); - let record_type = bytes.read_with(decode::read_u8)?; + let record_type = read_with(&mut bytes, decode::read_u8)?; match record_type { // 0 -> HistoryRecord::Create 0 => { // not super useful to us atm, but perhaps in the future // written by write_bin above - let _ = bytes.read_with(decode::read_bin_len)?; + let _ = read_with(&mut bytes, decode::read_bin_len)?; let record = History::deserialize(bytes.remaining_slice(), version)?; @@ -92,8 +92,8 @@ impl HistoryRecord { // 1 -> HistoryRecord::Delete 1 => { - let id = bytes.read_string()?; - bytes.expect_eof()?; + let id = read_string(&mut bytes)?; + expect_eof(&bytes)?; Ok(HistoryRecord::Delete(id.into())) } diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs index 0d40efc4f3c..bd34a3d1291 100644 --- a/crates/atuin-common/src/rmp.rs +++ b/crates/atuin-common/src/rmp.rs @@ -3,10 +3,12 @@ //! `rmp`'s own error types are awkward: some don't implement [`Display`] for //! every `E`, none say which variant they are, and the decode cursor offers no //! owned-string read, no nil-aware optional read, and no structural checks -//! (array length, end-of-input). This module fills those gaps with two -//! extension traits — [`RmpDecodeExt`] for reading a [`Bytes`] cursor and -//! [`RmpEncodeExt`] for writing — plus [`DecodeError`] / [`EncodeError`], which -//! carry legible messages and convert cleanly into [`eyre::Report`]. +//! (array length, end-of-input). This module fills those gaps with a set of +//! free functions — [`read_with`], [`read_string`], [`read_optional`], +//! [`read_array_len`], [`expect_array_len`] and [`expect_eof`] for reading a +//! [`Bytes`] cursor, and [`write_optional`] for writing — plus [`DecodeError`] / +//! [`EncodeError`], which carry legible messages and convert cleanly into +//! [`eyre::Report`]. //! //! Because every decode read returns [`DecodeError`], a decoder can use `?` //! throughout and convert once at the boundary, rather than hand-rolling a @@ -83,7 +85,7 @@ impl From> for DecodeError<'_, E> { impl DecodeError<'_, E> { /// If this is a type mismatch, the [`Marker`] found instead of the expected - /// type. [`RmpDecodeExt::read_optional`] uses this to recognise nil. + /// type. [`read_optional`] uses this to recognise nil. pub fn type_mismatch(&self) -> Option { match self { Self::DecodeString(DecodeStringError::TypeMismatch(m)) @@ -100,146 +102,105 @@ impl From> for eyre::Report { } } -/// Reading helpers for a MessagePack [`Bytes`] cursor. +/// Run an `rmp` decode function, converting its error into [`DecodeError`]. /// -/// Every method reports failures as [`DecodeError`], so decoders use `?` -/// throughout and convert once at the boundary. -pub trait RmpDecodeExt<'a> { - /// Run an `rmp` decode function, converting its error into [`DecodeError`]. - /// - /// Lets raw `rmp::decode` functions compose with `?`: - /// `bytes.read_with(rmp::decode::read_u64)?`. - fn read_with(&mut self, read: F) -> Result> - where - F: FnOnce(&mut Self) -> Result, - E: Into>; - - /// Read an owned [`String`]. - fn read_string(&mut self) -> Result>; - - /// Read a value that may be encoded as nil, returning [`None`] for nil. - /// - /// `read` decodes a `T`; if it fails specifically because it found - /// [`Marker::Null`], this yields [`None`] with the cursor left just past the - /// nil. Any other error is forwarded unchanged. - fn read_optional(&mut self, read: F) -> Result, DecodeError<'a>> - where - F: FnOnce(&mut Self) -> Result, - E: Into>; - - /// Read an array-length marker. - fn read_array_len(&mut self) -> Result>; - - /// Read an array-length marker and require it to equal `expected`. - /// - /// Returns [`DecodeError::UnexpectedArrayLen`] otherwise. For a - /// forward-compatible field count, use [`read_array_len`](Self::read_array_len) - /// and range-check the value yourself. - fn expect_array_len(&mut self, expected: u32) -> Result>; - - /// Succeed only if the cursor is at end-of-input, else - /// [`DecodeError::TrailingBytes`] — the standard malformed-record guard after - /// a fixed set of fields. - fn expect_eof(&self) -> Result<(), DecodeError<'a>>; +/// Lets raw `rmp::decode` functions compose with `?`: +/// `read_with(&mut bytes, rmp::decode::read_u64)?`. +pub fn read_with<'a, T, E>( + bytes: &mut Bytes<'a>, + read: impl FnOnce(&mut Bytes<'a>) -> Result, +) -> Result> +where + E: Into>, +{ + read(bytes).map_err(Into::into) } -impl<'a> RmpDecodeExt<'a> for Bytes<'a> { - fn read_with(&mut self, read: F) -> Result> - where - F: FnOnce(&mut Self) -> Result, - E: Into>, - { - read(self).map_err(Into::into) - } - - fn read_string(&mut self) -> Result> { - let slice = self.remaining_slice(); - let (string, rest) = match decode::read_str_from_slice(slice) { - Ok(pair) => pair, - Err(e) => { - if let DecodeStringError::TypeMismatch(_) = e { - // rmp's decode functions consume the marker byte on a type - // mismatch; do the same so `read_optional` can detect nil. - self.read_u8() - .expect("TypeMismatch implies the stream contains a marker byte"); - } - return Err(e.into()); - } - }; - *self = Bytes::new(rest); - Ok(string.into()) - } - - fn read_optional(&mut self, read: F) -> Result, DecodeError<'a>> - where - F: FnOnce(&mut Self) -> Result, - E: Into>, - { - match read(self) { - Ok(v) => Ok(Some(v)), - Err(e) => { - let e = e.into(); - if let Some(Marker::Null) = e.type_mismatch() { - Ok(None) - } else { - Err(e) - } +/// Read an owned [`String`]. +pub fn read_string<'a>(bytes: &mut Bytes<'a>) -> Result> { + let slice = bytes.remaining_slice(); + let (string, rest) = match decode::read_str_from_slice(slice) { + Ok(pair) => pair, + Err(e) => { + if let DecodeStringError::TypeMismatch(_) = e { + // rmp's decode functions consume the marker byte on a type + // mismatch; do the same so `read_optional` can detect nil. + bytes + .read_u8() + .expect("TypeMismatch implies the stream contains a marker byte"); } + return Err(e.into()); } - } - - fn read_array_len(&mut self) -> Result> { - decode::read_array_len(self).map_err(Into::into) - } + }; + *bytes = Bytes::new(rest); + Ok(string.into()) +} - fn expect_array_len(&mut self, expected: u32) -> Result> { - let actual = self.read_array_len()?; - if actual == expected { - Ok(actual) - } else { - Err(DecodeError::UnexpectedArrayLen { expected, actual }) +/// Read a value that may be encoded as nil, returning [`None`] for nil. +/// +/// `read` decodes a `T`; if it fails specifically because it found +/// [`Marker::Null`], this yields [`None`] with the cursor left just past the +/// nil. Any other error is forwarded unchanged. +pub fn read_optional<'a, T, E>( + bytes: &mut Bytes<'a>, + read: impl FnOnce(&mut Bytes<'a>) -> Result, +) -> Result, DecodeError<'a>> +where + E: Into>, +{ + match read(bytes) { + Ok(v) => Ok(Some(v)), + Err(e) => { + let e = e.into(); + if let Some(Marker::Null) = e.type_mismatch() { + Ok(None) + } else { + Err(e) + } } } +} - fn expect_eof(&self) -> Result<(), DecodeError<'a>> { - let remaining = self.remaining_slice().len(); - if remaining == 0 { - Ok(()) - } else { - Err(DecodeError::TrailingBytes { remaining }) - } +/// Read an array-length marker. +pub fn read_array_len<'a>(bytes: &mut Bytes<'a>) -> Result> { + decode::read_array_len(bytes).map_err(Into::into) +} + +/// Read an array-length marker and require it to equal `expected`. +/// +/// Returns [`DecodeError::UnexpectedArrayLen`] otherwise. For a +/// forward-compatible field count, use [`read_array_len`] and range-check the +/// value yourself. +pub fn expect_array_len<'a>(bytes: &mut Bytes<'a>, expected: u32) -> Result> { + let actual = read_array_len(bytes)?; + if actual == expected { + Ok(actual) + } else { + Err(DecodeError::UnexpectedArrayLen { expected, actual }) } } -/// Writing helpers for a MessagePack output buffer. -pub trait RmpEncodeExt: RmpWrite { - /// Write an optional value, encoding [`None`] as [`Marker::Null`]. - /// - /// The mirror of [`RmpDecodeExt::read_optional`]: `write` encodes the inner - /// value when `value` is [`Some`]. - fn write_optional( - &mut self, - value: Option, - write: F, - ) -> Result<(), EncodeError> - where - F: FnOnce(&mut Self, T) -> Result<(), ValueWriteError>; +/// Succeed only if the cursor is at end-of-input, else +/// [`DecodeError::TrailingBytes`]. +pub fn expect_eof<'a>(bytes: &Bytes<'a>) -> Result<(), DecodeError<'a>> { + let remaining = bytes.remaining_slice().len(); + if remaining == 0 { + Ok(()) + } else { + Err(DecodeError::TrailingBytes { remaining }) + } } -impl RmpEncodeExt for W { - fn write_optional( - &mut self, - value: Option, - write: F, - ) -> Result<(), EncodeError> - where - F: FnOnce(&mut Self, T) -> Result<(), ValueWriteError>, - { - match value { - Some(v) => write(self, v).map_err(EncodeError::from), - None => encode::write_nil(self) - .map_err(|e| EncodeError::from(ValueWriteError::InvalidMarkerWrite(e))), - } +/// Write an optional value, encoding [`None`] as [`Marker::Null`]. +pub fn write_optional( + out: &mut W, + value: Option, + write: impl FnOnce(&mut W, T) -> Result<(), ValueWriteError>, +) -> Result<(), EncodeError> { + match value { + Some(v) => write(out, v).map_err(EncodeError::from), + None => encode::write_nil(out) + .map_err(|e| EncodeError::from(ValueWriteError::InvalidMarkerWrite(e))), } } @@ -310,7 +271,7 @@ mod tests { fn read_string_round_trips() { let buf = enc(|v| rmp::encode::write_str(v, "héllo 🦀").unwrap()); let mut bytes = Bytes::new(&buf); - assert_eq!(bytes.read_string().unwrap(), "héllo 🦀"); + assert_eq!(read_string(&mut bytes).unwrap(), "héllo 🦀"); assert!(bytes.remaining_slice().is_empty()); } @@ -319,7 +280,7 @@ mod tests { // A lone nil marker: read_string must fail *and* consume the marker so a // following read_optional can observe end-of-input correctly. let mut bytes = Bytes::new(&[0xc0]); - assert!(bytes.read_string().is_err()); + assert!(read_string(&mut bytes).is_err()); assert!( bytes.remaining_slice().is_empty(), "marker byte must be consumed" @@ -330,18 +291,21 @@ mod tests { fn read_with_converts_rmp_errors() { let buf = enc(|v| rmp::encode::write_u64(v, 42).unwrap()); let mut bytes = Bytes::new(&buf); - assert_eq!(bytes.read_with(rmp::decode::read_u64).unwrap(), 42); + assert_eq!(read_with(&mut bytes, rmp::decode::read_u64).unwrap(), 42); } #[test] fn read_optional_some_and_none() { let some = enc(|v| rmp::encode::write_u64(v, 7).unwrap()); let mut b = Bytes::new(&some); - assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), Some(7)); + assert_eq!( + read_optional(&mut b, rmp::decode::read_u64).unwrap(), + Some(7) + ); let none = enc(|v| rmp::encode::write_nil(v).unwrap()); let mut b = Bytes::new(&none); - assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), None); + assert_eq!(read_optional(&mut b, rmp::decode::read_u64).unwrap(), None); assert!(b.remaining_slice().is_empty()); } @@ -350,7 +314,7 @@ mod tests { let buf = enc(|v| rmp::encode::write_str(v, "x").unwrap()); let mut b = Bytes::new(&buf); assert_eq!( - b.read_optional(|b| b.read_string()).unwrap(), + read_optional(&mut b, read_string).unwrap(), Some("x".to_string()) ); } @@ -360,7 +324,7 @@ mod tests { // A bool where a u64 is expected is a type mismatch that is NOT nil. let buf = enc(|v| rmp::encode::write_bool(v, true).unwrap()); let mut b = Bytes::new(&buf); - assert!(b.read_optional(rmp::decode::read_u64).is_err()); + assert!(read_optional(&mut b, rmp::decode::read_u64).is_err()); } fn array_of(len: u32) -> Vec { @@ -373,14 +337,14 @@ mod tests { fn expect_array_len_exact_ok() { let buf = array_of(3); let mut b = Bytes::new(&buf); - assert_eq!(b.expect_array_len(3).unwrap(), 3); + assert_eq!(expect_array_len(&mut b, 3).unwrap(), 3); } #[test] fn expect_array_len_mismatch_reports_expected_and_actual() { let buf = array_of(5); let mut b = Bytes::new(&buf); - match b.expect_array_len(3) { + match expect_array_len(&mut b, 3) { Err(DecodeError::UnexpectedArrayLen { expected, actual }) => { assert_eq!((expected, actual), (3, 5)); } @@ -392,21 +356,21 @@ mod tests { fn read_array_len_returns_count_for_manual_range_checks() { let buf = array_of(9); let mut b = Bytes::new(&buf); - assert_eq!(b.read_array_len().unwrap(), 9); + assert_eq!(read_array_len(&mut b).unwrap(), 9); } #[test] fn expect_eof_ok_when_consumed() { let buf = enc(|v| rmp::encode::write_u8(v, 1).unwrap()); let mut b = Bytes::new(&buf); - b.read_with(rmp::decode::read_u8).unwrap(); - assert!(b.expect_eof().is_ok()); + read_with(&mut b, rmp::decode::read_u8).unwrap(); + assert!(expect_eof(&b).is_ok()); } #[test] fn expect_eof_reports_remaining() { let b = Bytes::new(&[0x01, 0x02, 0x03]); - match b.expect_eof() { + match expect_eof(&b) { Err(DecodeError::TrailingBytes { remaining }) => assert_eq!(remaining, 3), other => panic!("expected TrailingBytes, got {other:?}"), } @@ -415,29 +379,29 @@ mod tests { #[test] fn write_optional_some_then_read_back() { let mut out = Vec::new(); - out.write_optional(Some(99u64), rmp::encode::write_u64) - .unwrap(); + write_optional(&mut out, Some(99u64), rmp::encode::write_u64).unwrap(); let mut b = Bytes::new(&out); - assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), Some(99)); + assert_eq!( + read_optional(&mut b, rmp::decode::read_u64).unwrap(), + Some(99) + ); } #[test] fn write_optional_none_writes_nil() { let mut out = Vec::new(); - out.write_optional::(None, rmp::encode::write_u64) - .unwrap(); + write_optional::<_, u64>(&mut out, None, rmp::encode::write_u64).unwrap(); let mut b = Bytes::new(&out); - assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), None); + assert_eq!(read_optional(&mut b, rmp::decode::read_u64).unwrap(), None); } #[test] fn write_optional_str() { let mut out = Vec::new(); - out.write_optional(Some("hi"), rmp::encode::write_str) - .unwrap(); + write_optional(&mut out, Some("hi"), rmp::encode::write_str).unwrap(); let mut b = Bytes::new(&out); assert_eq!( - b.read_optional(|b| b.read_string()).unwrap(), + read_optional(&mut b, read_string).unwrap(), Some("hi".to_string()) ); } @@ -447,14 +411,13 @@ mod tests { // A nil optional-string field followed by a u64. The nil read must consume // exactly the marker so the following field decodes correctly. let mut out = Vec::new(); - out.write_optional::<&str, _>(None, rmp::encode::write_str) - .unwrap(); + write_optional::<_, &str>(&mut out, None, rmp::encode::write_str).unwrap(); rmp::encode::write_u64(&mut out, 1234).unwrap(); let mut b = Bytes::new(&out); - assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), None); - assert_eq!(b.read_with(rmp::decode::read_u64).unwrap(), 1234); - assert!(b.expect_eof().is_ok()); + assert_eq!(read_optional(&mut b, read_string).unwrap(), None); + assert_eq!(read_with(&mut b, rmp::decode::read_u64).unwrap(), 1234); + assert!(expect_eof(&b).is_ok()); } use proptest::prelude::*; @@ -467,7 +430,7 @@ mod tests { fn string_round_trips(s in r"(?s).*") { let buf = enc(|v| rmp::encode::write_str(v, &s).unwrap()); let mut b = Bytes::new(&buf); - prop_assert_eq!(b.read_string().unwrap(), s); + prop_assert_eq!(read_string(&mut b).unwrap(), s); prop_assert!(b.remaining_slice().is_empty()); } @@ -475,18 +438,18 @@ mod tests { #[test] fn optional_u64_round_trips(v in proptest::option::of(any::())) { let mut out = Vec::new(); - out.write_optional(v, rmp::encode::write_u64).unwrap(); + write_optional(&mut out, v, rmp::encode::write_u64).unwrap(); let mut b = Bytes::new(&out); - prop_assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), v); + prop_assert_eq!(read_optional(&mut b, rmp::decode::read_u64).unwrap(), v); } // Option survives the closure-based optional round trip. #[test] fn optional_string_round_trips(v in proptest::option::of(r"(?s).*")) { let mut out = Vec::new(); - out.write_optional(v.as_deref(), rmp::encode::write_str).unwrap(); + write_optional(&mut out, v.as_deref(), rmp::encode::write_str).unwrap(); let mut b = Bytes::new(&out); - prop_assert_eq!(b.read_optional(|b| b.read_string()).unwrap(), v); + prop_assert_eq!(read_optional(&mut b, read_string).unwrap(), v); prop_assert!(b.remaining_slice().is_empty()); } @@ -502,25 +465,25 @@ mod tests { rmp::encode::write_array_len(&mut out, 3).unwrap(); rmp::encode::write_str(&mut out, &id).unwrap(); rmp::encode::write_u64(&mut out, ts).unwrap(); - out.write_optional(deleted, rmp::encode::write_u64).unwrap(); + write_optional(&mut out, deleted, rmp::encode::write_u64).unwrap(); let mut b = Bytes::new(&out); - prop_assert_eq!(b.expect_array_len(3).unwrap(), 3); - prop_assert_eq!(b.read_string().unwrap(), id); - prop_assert_eq!(b.read_with(rmp::decode::read_u64).unwrap(), ts); - prop_assert_eq!(b.read_optional(rmp::decode::read_u64).unwrap(), deleted); - prop_assert!(b.expect_eof().is_ok()); + prop_assert_eq!(expect_array_len(&mut b, 3).unwrap(), 3); + prop_assert_eq!(read_string(&mut b).unwrap(), id); + prop_assert_eq!(read_with(&mut b, rmp::decode::read_u64).unwrap(), ts); + prop_assert_eq!(read_optional(&mut b, rmp::decode::read_u64).unwrap(), deleted); + prop_assert!(expect_eof(&b).is_ok()); } // Reads never panic on arbitrary bytes — they return Err instead. #[test] fn reads_never_panic(raw in proptest::collection::vec(any::(), 0..64)) { let mut b = Bytes::new(&raw); - let _ = b.read_string(); + let _ = read_string(&mut b); let mut b = Bytes::new(&raw); - let _ = b.read_array_len(); + let _ = read_array_len(&mut b); let mut b = Bytes::new(&raw); - let _ = b.read_optional(rmp::decode::read_u64); + let _ = read_optional(&mut b, rmp::decode::read_u64); } } } diff --git a/crates/atuin-dotfiles/src/shell.rs b/crates/atuin-dotfiles/src/shell.rs index 02b8fbdaeb9..87ccba93946 100644 --- a/crates/atuin-dotfiles/src/shell.rs +++ b/crates/atuin-dotfiles/src/shell.rs @@ -2,7 +2,7 @@ use eyre::Result; use rmp::{decode, encode}; use serde::Serialize; -use atuin_common::rmp::RmpDecodeExt as _; +use atuin_common::rmp::{expect_array_len, expect_eof, read_string, read_with}; use atuin_common::shell::{Shell, ShellError}; use crate::store::AliasStore; @@ -43,13 +43,13 @@ impl Var { } pub fn deserialize(bytes: &mut decode::Bytes) -> Result { - bytes.expect_array_len(3)?; + expect_array_len(bytes, 3)?; - let name = bytes.read_string()?; - let value = bytes.read_string()?; - let export = bytes.read_with(decode::read_bool)?; + let name = read_string(bytes)?; + let value = read_string(bytes)?; + let export = read_with(bytes, decode::read_bool)?; - bytes.expect_eof()?; + expect_eof(bytes)?; Ok(Var { name, diff --git a/crates/atuin-dotfiles/src/store.rs b/crates/atuin-dotfiles/src/store.rs index 53486cd2c53..fac79626132 100644 --- a/crates/atuin-dotfiles/src/store.rs +++ b/crates/atuin-dotfiles/src/store.rs @@ -6,7 +6,7 @@ use atuin_client::record::sqlite_store::SqliteStore; // While we will support a range of shell config, I'd rather have a larger number of small records // + stores, rather than one mega config store. use atuin_common::record::{DecryptedData, Host, HostId}; -use atuin_common::rmp::RmpDecodeExt as _; +use atuin_common::rmp::{expect_array_len, expect_eof, read_string, read_with}; use atuin_common::utils::unquote; use eyre::{Result, bail, eyre}; @@ -60,28 +60,28 @@ impl AliasRecord { CONFIG_SHELL_ALIAS_VERSION => { let mut bytes = decode::Bytes::new(&data.0); - let record_type = bytes.read_with(decode::read_u8)?; + let record_type = read_with(&mut bytes, decode::read_u8)?; match record_type { // create 0 => { - bytes.expect_array_len(2)?; + expect_array_len(&mut bytes, 2)?; - let name = bytes.read_string()?; - let value = bytes.read_string()?; + let name = read_string(&mut bytes)?; + let value = read_string(&mut bytes)?; - bytes.expect_eof()?; + expect_eof(&bytes)?; Ok(AliasRecord::Create(Alias { name, value })) } // delete 1 => { - bytes.expect_array_len(1)?; + expect_array_len(&mut bytes, 1)?; - let key = bytes.read_string()?; + let key = read_string(&mut bytes)?; - bytes.expect_eof()?; + expect_eof(&bytes)?; Ok(AliasRecord::Delete(key)) } diff --git a/crates/atuin-dotfiles/src/store/var.rs b/crates/atuin-dotfiles/src/store/var.rs index 3d329a32e53..ec2cc723878 100644 --- a/crates/atuin-dotfiles/src/store/var.rs +++ b/crates/atuin-dotfiles/src/store/var.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; use atuin_client::record::sqlite_store::SqliteStore; use atuin_common::record::{DecryptedData, Host, HostId}; -use atuin_common::rmp::RmpDecodeExt as _; +use atuin_common::rmp::{expect_array_len, expect_eof, read_string, read_with}; use eyre::{Result, bail, eyre}; use atuin_client::record::encryption::PASETO_V4; @@ -54,7 +54,7 @@ impl VarRecord { DOTFILES_VAR_VERSION => { let mut bytes = decode::Bytes::new(&data.0); - let record_type = bytes.read_with(decode::read_u8)?; + let record_type = read_with(&mut bytes, decode::read_u8)?; match record_type { // create @@ -65,11 +65,11 @@ impl VarRecord { // delete 1 => { - bytes.expect_array_len(1)?; + expect_array_len(&mut bytes, 1)?; - let key = bytes.read_string()?; + let key = read_string(&mut bytes)?; - bytes.expect_eof()?; + expect_eof(&bytes)?; Ok(VarRecord::Delete(key)) } diff --git a/crates/atuin-kv/src/store/record.rs b/crates/atuin-kv/src/store/record.rs index f41e03ccaa2..b48d2ddf749 100644 --- a/crates/atuin-kv/src/store/record.rs +++ b/crates/atuin-kv/src/store/record.rs @@ -1,5 +1,5 @@ use atuin_common::record::DecryptedData; -use atuin_common::rmp::RmpDecodeExt as _; +use atuin_common::rmp::{expect_array_len, expect_eof, read_string, read_with}; use eyre::{Result, bail}; use typed_builder::TypedBuilder; @@ -41,13 +41,13 @@ impl KvRecord { "v0" => { let mut bytes = decode::Bytes::new(&data.0); - bytes.expect_array_len(3)?; + expect_array_len(&mut bytes, 3)?; - let namespace = bytes.read_string()?; - let key = bytes.read_string()?; - let value = bytes.read_string()?; + let namespace = read_string(&mut bytes)?; + let key = read_string(&mut bytes)?; + let value = read_string(&mut bytes)?; - bytes.expect_eof()?; + expect_eof(&bytes)?; Ok(KvRecord { namespace, @@ -58,19 +58,19 @@ impl KvRecord { KV_VERSION => { let mut bytes = decode::Bytes::new(&data.0); - bytes.expect_array_len(4)?; + expect_array_len(&mut bytes, 4)?; - let namespace = bytes.read_string()?; - let key = bytes.read_string()?; - let has_value = bytes.read_with(decode::read_bool)?; + let namespace = read_string(&mut bytes)?; + let key = read_string(&mut bytes)?; + let has_value = read_with(&mut bytes, decode::read_bool)?; let value = if has_value { - Some(bytes.read_string()?) + Some(read_string(&mut bytes)?) } else { None }; - bytes.expect_eof()?; + expect_eof(&bytes)?; Ok(KvRecord { namespace, diff --git a/crates/atuin-scripts/src/store/record.rs b/crates/atuin-scripts/src/store/record.rs index bc3f12d6e64..63e35165385 100644 --- a/crates/atuin-scripts/src/store/record.rs +++ b/crates/atuin-scripts/src/store/record.rs @@ -47,34 +47,34 @@ impl ScriptRecord { } pub fn deserialize(data: &DecryptedData, version: &str) -> Result { - use atuin_common::rmp::RmpDecodeExt as _; + use atuin_common::rmp::{read_string, read_with}; use rmp::decode; match version { SCRIPT_VERSION => { let mut bytes = decode::Bytes::new(&data.0); - let record_type = bytes.read_with(decode::read_u8)?; + let record_type = read_with(&mut bytes, decode::read_u8)?; match record_type { // create 0 => { // written by encode::write_bin above - let _ = bytes.read_with(decode::read_bin_len)?; + let _ = read_with(&mut bytes, decode::read_bin_len)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Create(script)) } // delete 1 => { - let id = bytes.read_string()?; + let id = read_string(&mut bytes)?; Ok(ScriptRecord::Delete(Uuid::parse_str(&id)?)) } // update 2 => { // written by encode::write_bin above - let _ = bytes.read_with(decode::read_bin_len)?; + let _ = read_with(&mut bytes, decode::read_bin_len)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Update(script)) } diff --git a/crates/atuin-scripts/src/store/script.rs b/crates/atuin-scripts/src/store/script.rs index 1c05a008cb0..fdf13b9c5ff 100644 --- a/crates/atuin-scripts/src/store/script.rs +++ b/crates/atuin-scripts/src/store/script.rs @@ -60,28 +60,28 @@ impl Script { } pub fn deserialize(bytes: &[u8]) -> Result { - use atuin_common::rmp::RmpDecodeExt as _; + use atuin_common::rmp::{expect_array_len, expect_eof, read_array_len, read_string}; let mut bytes = decode::Bytes::new(bytes); - bytes.expect_array_len(6)?; + expect_array_len(&mut bytes, 6)?; - let id = bytes.read_string()?; - let name = bytes.read_string()?; - let description = bytes.read_string()?; - let shebang = bytes.read_string()?; + let id = read_string(&mut bytes)?; + let name = read_string(&mut bytes)?; + let description = read_string(&mut bytes)?; + let shebang = read_string(&mut bytes)?; - let tags_len = bytes.read_array_len()?; + let tags_len = read_array_len(&mut bytes)?; let mut tags = Vec::new(); for _ in 0..tags_len { - let tag = bytes.read_string()?; + let tag = read_string(&mut bytes)?; tags.push(tag); } - let script = bytes.read_string()?; + let script = read_string(&mut bytes)?; - bytes.expect_eof()?; + expect_eof(&bytes)?; Ok(Script { id: Uuid::parse_str(&id)?, From 1cc1150efaa604234e06789d2349516aaa663f41 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 23:34:54 -0700 Subject: [PATCH 18/27] refactor(common): generic decode::() via Decode trait for rmp helpers --- crates/atuin-client/src/encryption.rs | 10 +- crates/atuin-client/src/history.rs | 33 ++- crates/atuin-client/src/history/store.rs | 11 +- crates/atuin-common/src/rmp.rs | 271 +++++++++++++---------- crates/atuin-dotfiles/src/shell.rs | 12 +- crates/atuin-dotfiles/src/store.rs | 14 +- crates/atuin-dotfiles/src/store/var.rs | 10 +- crates/atuin-kv/src/store/record.rs | 22 +- crates/atuin-scripts/src/store/record.rs | 13 +- crates/atuin-scripts/src/store/script.rs | 20 +- 10 files changed, 225 insertions(+), 191 deletions(-) diff --git a/crates/atuin-client/src/encryption.rs b/crates/atuin-client/src/encryption.rs index e8b54652f21..d973caee165 100644 --- a/crates/atuin-client/src/encryption.rs +++ b/crates/atuin-client/src/encryption.rs @@ -17,7 +17,7 @@ use eyre::{Context, Result, bail, ensure}; use fs_err as fs; use rmp::Marker; -use atuin_common::rmp::{expect_array_len, read_with}; +use atuin_common::rmp::{Bytes, decode, decode_bin_len, expect_array_len}; use crate::settings::Settings; @@ -71,8 +71,6 @@ pub fn encode_key(key: &Key) -> Result { } pub fn decode_key(key: String) -> Result { - use rmp::decode; - let buf = BASE64_STANDARD .decode(key.trim_end()) .wrap_err("encryption key is not a valid base64 encoding")?; @@ -82,11 +80,11 @@ pub fn decode_key(key: String) -> Result { match <[u8; 32]>::try_from(&*buf) { Ok(key) => Ok(key.into()), Err(_) => { - let mut bytes = rmp::decode::Bytes::new(&buf); + let mut bytes = Bytes::new(&buf); match Marker::from_u8(buf[0]) { Marker::Bin8 => { - let len = read_with(&mut bytes, decode::read_bin_len)?; + let len = decode_bin_len(&mut bytes)?; ensure!(len == 32, "encryption key is not the correct size"); let key = <[u8; 32]>::try_from(bytes.remaining_slice()) .context("could not decode encryption key")?; @@ -97,7 +95,7 @@ pub fn decode_key(key: String) -> Result { let mut key = Key::default(); for i in &mut key { - *i = read_with(&mut bytes, decode::read_int)?; + *i = decode::(&mut bytes)?; } Ok(key) } diff --git a/crates/atuin-client/src/history.rs b/crates/atuin-client/src/history.rs index e9c4cbc03a4..44da29a682d 100644 --- a/crates/atuin-client/src/history.rs +++ b/crates/atuin-client/src/history.rs @@ -1,4 +1,3 @@ -use rmp::decode::{self, Bytes}; use rmp::encode; use std::env; @@ -10,9 +9,7 @@ use eyre::{Result, bail}; use crate::secrets::SECRET_PATTERNS_RE; use crate::settings::Settings; use crate::utils::get_host_user; -use atuin_common::rmp::{ - EncodeError, read_array_len, read_optional, read_string, read_with, write_optional, -}; +use atuin_common::rmp::{Bytes, EncodeError, decode, decode_array_len, write_optional}; use time::OffsetDateTime; pub(crate) mod builder; @@ -250,30 +247,30 @@ impl History { let mut bytes = Bytes::new(bytes); - let real_version = read_with(&mut bytes, decode::read_u16)?; + let real_version = decode::(&mut bytes)?; if real_version != version.as_int() { bail!("expected to decode {version} record, found v{real_version}"); } - let nfields = read_array_len(&mut bytes)?; + let nfields = decode_array_len(&mut bytes)?; let min_fields = version.min_fields(); if nfields < min_fields || version.max_fields().is_some_and(|max| nfields > max) { bail!("unexpected number of fields ({nfields}) for history version {version}"); } - let id = read_string(&mut bytes)?; - let timestamp = read_with(&mut bytes, decode::read_u64)?; - let duration = read_with(&mut bytes, decode::read_int)?; - let exit = read_with(&mut bytes, decode::read_int)?; + let id = decode::(&mut bytes)?; + let timestamp = decode::(&mut bytes)?; + let duration = decode::(&mut bytes)?; + let exit = decode::(&mut bytes)?; - let command = read_string(&mut bytes)?; - let cwd = read_string(&mut bytes)?; - let session = read_string(&mut bytes)?; - let hostname = read_string(&mut bytes)?; - let deleted_at = read_optional(&mut bytes, decode::read_u64)?; + let command = decode::(&mut bytes)?; + let cwd = decode::(&mut bytes)?; + let session = decode::(&mut bytes)?; + let hostname = decode::(&mut bytes)?; + let deleted_at = decode::>(&mut bytes)?; let author = if version >= Version::One { - read_optional(&mut bytes, read_string)? + decode::>(&mut bytes)? } else { None }; @@ -283,13 +280,13 @@ impl History { Version::One => nfields > min_fields, Version::Two => true, } { - read_optional(&mut bytes, read_string)? + decode::>(&mut bytes)? } else { None }; let shell = if version >= Version::Two { - read_optional(&mut bytes, read_string)? + decode::>(&mut bytes)? } else { None }; diff --git a/crates/atuin-client/src/history/store.rs b/crates/atuin-client/src/history/store.rs index 5309c5187ce..883defbf796 100644 --- a/crates/atuin-client/src/history/store.rs +++ b/crates/atuin-client/src/history/store.rs @@ -3,14 +3,13 @@ use std::{collections::HashSet, fmt::Write, time::Duration}; use eyre::{Result, bail, eyre}; use futures::{Stream, StreamExt, TryStreamExt, future, stream}; use indicatif::{ProgressBar, ProgressState, ProgressStyle}; -use rmp::decode::Bytes; use crate::{ database::{Database, current_context}, record::{encryption::PASETO_V4, sqlite_store::SqliteStore, store::Store}, }; use atuin_common::record::{DecryptedData, Host, HostId, Record, RecordId, RecordIdx}; -use atuin_common::rmp::{expect_eof, read_string, read_with}; +use atuin_common::rmp::{Bytes, decode, decode_bin_len, expect_eof}; use super::{HISTORY_TAG, History, HistoryId, Version}; @@ -72,18 +71,16 @@ impl HistoryRecord { } pub fn deserialize(bytes: &DecryptedData, version: &str) -> Result { - use rmp::decode; - let mut bytes = Bytes::new(&bytes.0); - let record_type = read_with(&mut bytes, decode::read_u8)?; + let record_type = decode::(&mut bytes)?; match record_type { // 0 -> HistoryRecord::Create 0 => { // not super useful to us atm, but perhaps in the future // written by write_bin above - let _ = read_with(&mut bytes, decode::read_bin_len)?; + let _ = decode_bin_len(&mut bytes)?; let record = History::deserialize(bytes.remaining_slice(), version)?; @@ -92,7 +89,7 @@ impl HistoryRecord { // 1 -> HistoryRecord::Delete 1 => { - let id = read_string(&mut bytes)?; + let id = decode::(&mut bytes)?; expect_eof(&bytes)?; Ok(HistoryRecord::Delete(id.into())) diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs index bd34a3d1291..4b698037640 100644 --- a/crates/atuin-common/src/rmp.rs +++ b/crates/atuin-common/src/rmp.rs @@ -3,12 +3,15 @@ //! `rmp`'s own error types are awkward: some don't implement [`Display`] for //! every `E`, none say which variant they are, and the decode cursor offers no //! owned-string read, no nil-aware optional read, and no structural checks -//! (array length, end-of-input). This module fills those gaps with a set of -//! free functions — [`read_with`], [`read_string`], [`read_optional`], -//! [`read_array_len`], [`expect_array_len`] and [`expect_eof`] for reading a -//! [`Bytes`] cursor, and [`write_optional`] for writing — plus [`DecodeError`] / -//! [`EncodeError`], which carry legible messages and convert cleanly into -//! [`eyre::Report`]. +//! (array length, end-of-input). This module fills those gaps. +//! +//! Values are read through the generic [`decode()`] function, backed by the +//! [`Decode`] trait: `decode::(&mut bytes)?`, +//! `decode::>(&mut bytes)?`. Structural markers that aren't values +//! have their own functions — [`decode_array_len`], [`decode_bin_len`], +//! [`expect_array_len`] and [`expect_eof`] — and [`write_optional`] handles the +//! encode side. All of these return [`DecodeError`] / [`EncodeError`], which +//! carry legible messages and convert cleanly into [`eyre::Report`]. //! //! Because every decode read returns [`DecodeError`], a decoder can use `?` //! throughout and convert once at the boundary, rather than hand-rolling a @@ -17,12 +20,15 @@ //! [`Display`]: std::fmt::Display use rmp::Marker; -use rmp::decode::bytes::{Bytes, BytesReadError}; +use rmp::decode::bytes::BytesReadError; use rmp::decode::{ self, DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError, }; use rmp::encode::{self, RmpWrite, RmpWriteErr, ValueWriteError}; +/// Re-exported so decoders need only depend on this module, not `rmp::decode`. +pub use rmp::decode::bytes::Bytes; + /// An error encountered while encoding a MessagePack value. /// /// Wraps [`ValueWriteError`] with a message that names the failing write and its @@ -85,7 +91,8 @@ impl From> for DecodeError<'_, E> { impl DecodeError<'_, E> { /// If this is a type mismatch, the [`Marker`] found instead of the expected - /// type. [`read_optional`] uses this to recognise nil. + /// type. The [`Option`] implementation of [`Decode`] uses this to recognise + /// nil. pub fn type_mismatch(&self) -> Option { match self { Self::DecodeString(DecodeStringError::TypeMismatch(m)) @@ -102,77 +109,91 @@ impl From> for eyre::Report { } } -/// Run an `rmp` decode function, converting its error into [`DecodeError`]. +/// A value decodable from a MessagePack [`Bytes`] cursor. /// -/// Lets raw `rmp::decode` functions compose with `?`: -/// `read_with(&mut bytes, rmp::decode::read_u64)?`. -pub fn read_with<'a, T, E>( - bytes: &mut Bytes<'a>, - read: impl FnOnce(&mut Bytes<'a>) -> Result, -) -> Result> -where - E: Into>, -{ - read(bytes).map_err(Into::into) +/// Implemented for the scalars atuin's record formats use and for [`Option`] +/// (nil ⇒ [`None`]). Call it through the generic [`decode()`] function; structural +/// markers that aren't values have their own functions ([`decode_array_len`], +/// [`decode_bin_len`]). +pub trait Decode: Sized { + fn decode<'a>(bytes: &mut Bytes<'a>) -> Result>; } -/// Read an owned [`String`]. -pub fn read_string<'a>(bytes: &mut Bytes<'a>) -> Result> { - let slice = bytes.remaining_slice(); - let (string, rest) = match decode::read_str_from_slice(slice) { - Ok(pair) => pair, - Err(e) => { - if let DecodeStringError::TypeMismatch(_) = e { - // rmp's decode functions consume the marker byte on a type - // mismatch; do the same so `read_optional` can detect nil. - bytes - .read_u8() - .expect("TypeMismatch implies the stream contains a marker byte"); +/// Decode a [`Decode`] value: `decode::(&mut bytes)?`, +/// `decode::>(&mut bytes)?`. +pub fn decode<'a, T: Decode>(bytes: &mut Bytes<'a>) -> Result> { + T::decode(bytes) +} + +impl Decode for String { + fn decode<'a>(bytes: &mut Bytes<'a>) -> Result> { + let slice = bytes.remaining_slice(); + let (string, rest) = match decode::read_str_from_slice(slice) { + Ok(pair) => pair, + Err(e) => { + if let DecodeStringError::TypeMismatch(_) = e { + // rmp consumes the marker byte on a type mismatch; match that + // so `Option::decode` can detect nil. + bytes + .read_u8() + .expect("TypeMismatch implies the stream contains a marker byte"); + } + return Err(e.into()); } - return Err(e.into()); - } - }; - *bytes = Bytes::new(rest); - Ok(string.into()) + }; + *bytes = Bytes::new(rest); + Ok(string.into()) + } } -/// Read a value that may be encoded as nil, returning [`None`] for nil. -/// -/// `read` decodes a `T`; if it fails specifically because it found -/// [`Marker::Null`], this yields [`None`] with the cursor left just past the -/// nil. Any other error is forwarded unchanged. -pub fn read_optional<'a, T, E>( - bytes: &mut Bytes<'a>, - read: impl FnOnce(&mut Bytes<'a>) -> Result, -) -> Result, DecodeError<'a>> -where - E: Into>, -{ - match read(bytes) { - Ok(v) => Ok(Some(v)), - Err(e) => { - let e = e.into(); - if let Some(Marker::Null) = e.type_mismatch() { - Ok(None) - } else { - Err(e) +impl Decode for bool { + fn decode<'a>(bytes: &mut Bytes<'a>) -> Result> { + decode::read_bool(bytes).map_err(Into::into) + } +} + +macro_rules! impl_decode_int { + ($($t:ty),+ $(,)?) => {$( + impl Decode for $t { + fn decode<'a>(bytes: &mut Bytes<'a>) -> Result> { + decode::read_int(bytes).map_err(Into::into) + } + } + )+}; +} +impl_decode_int!(u8, u16, u64, i64); + +impl Decode for Option { + /// Decodes a `T`, mapping a nil marker to [`None`]. + fn decode<'a>(bytes: &mut Bytes<'a>) -> Result> { + match T::decode(bytes) { + Ok(v) => Ok(Some(v)), + Err(e) => { + if let Some(Marker::Null) = e.type_mismatch() { + Ok(None) + } else { + Err(e) + } } } } } -/// Read an array-length marker. -pub fn read_array_len<'a>(bytes: &mut Bytes<'a>) -> Result> { +/// Decode an array-length marker (the header count, not a value). +pub fn decode_array_len<'a>(bytes: &mut Bytes<'a>) -> Result> { decode::read_array_len(bytes).map_err(Into::into) } -/// Read an array-length marker and require it to equal `expected`. -/// -/// Returns [`DecodeError::UnexpectedArrayLen`] otherwise. For a -/// forward-compatible field count, use [`read_array_len`] and range-check the -/// value yourself. +/// Decode a binary-length marker (the header, before the raw payload bytes). +pub fn decode_bin_len<'a>(bytes: &mut Bytes<'a>) -> Result> { + decode::read_bin_len(bytes).map_err(Into::into) +} + +/// Decode an array-length marker and require it to equal `expected`, else +/// [`DecodeError::UnexpectedArrayLen`]. For a forward-compatible field count, +/// use [`decode_array_len`] and range-check yourself. pub fn expect_array_len<'a>(bytes: &mut Bytes<'a>, expected: u32) -> Result> { - let actual = read_array_len(bytes)?; + let actual = decode_array_len(bytes)?; if actual == expected { Ok(actual) } else { @@ -180,8 +201,7 @@ pub fn expect_array_len<'a>(bytes: &mut Bytes<'a>, expected: u32) -> Result(bytes: &Bytes<'a>) -> Result<(), DecodeError<'a>> { let remaining = bytes.remaining_slice().len(); if remaining == 0 { @@ -268,19 +288,19 @@ mod tests { } #[test] - fn read_string_round_trips() { + fn decode_string_round_trips() { let buf = enc(|v| rmp::encode::write_str(v, "héllo 🦀").unwrap()); let mut bytes = Bytes::new(&buf); - assert_eq!(read_string(&mut bytes).unwrap(), "héllo 🦀"); + assert_eq!(decode::(&mut bytes).unwrap(), "héllo 🦀"); assert!(bytes.remaining_slice().is_empty()); } #[test] - fn read_string_on_wrong_type_errors_and_consumes_marker() { - // A lone nil marker: read_string must fail *and* consume the marker so a - // following read_optional can observe end-of-input correctly. + fn decode_string_on_wrong_type_errors_and_consumes_marker() { + // A lone nil marker: decode:: must fail *and* consume the marker + // so a following decode::> can observe end-of-input correctly. let mut bytes = Bytes::new(&[0xc0]); - assert!(read_string(&mut bytes).is_err()); + assert!(decode::(&mut bytes).is_err()); assert!( bytes.remaining_slice().is_empty(), "marker byte must be consumed" @@ -288,43 +308,75 @@ mod tests { } #[test] - fn read_with_converts_rmp_errors() { + fn decode_converts_rmp_errors() { let buf = enc(|v| rmp::encode::write_u64(v, 42).unwrap()); let mut bytes = Bytes::new(&buf); - assert_eq!(read_with(&mut bytes, rmp::decode::read_u64).unwrap(), 42); + assert_eq!(decode::(&mut bytes).unwrap(), 42); + } + + #[rstest] + #[case::bool_true(enc(|v| rmp::encode::write_bool(v, true).unwrap()), true)] + #[case::bool_false(enc(|v| rmp::encode::write_bool(v, false).unwrap()), false)] + fn decode_bool_round_trips(#[case] buf: Vec, #[case] expected: bool) { + let mut b = Bytes::new(&buf); + assert_eq!(decode::(&mut b).unwrap(), expected); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn decode_u8_round_trips() { + let buf = enc(|v| rmp::encode::write_u8(v, 200).unwrap()); + let mut b = Bytes::new(&buf); + assert_eq!(decode::(&mut b).unwrap(), 200); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn decode_u16_round_trips() { + let buf = enc(|v| rmp::encode::write_u16(v, 40000).unwrap()); + let mut b = Bytes::new(&buf); + assert_eq!(decode::(&mut b).unwrap(), 40000); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn decode_i64_round_trips() { + let buf = enc(|v| { + rmp::encode::write_sint(v, -123456789).unwrap(); + }); + let mut b = Bytes::new(&buf); + assert_eq!(decode::(&mut b).unwrap(), -123456789); + assert!(b.remaining_slice().is_empty()); } #[test] - fn read_optional_some_and_none() { + fn decode_optional_some_and_none() { let some = enc(|v| rmp::encode::write_u64(v, 7).unwrap()); let mut b = Bytes::new(&some); - assert_eq!( - read_optional(&mut b, rmp::decode::read_u64).unwrap(), - Some(7) - ); + assert_eq!(decode::>(&mut b).unwrap(), Some(7)); let none = enc(|v| rmp::encode::write_nil(v).unwrap()); let mut b = Bytes::new(&none); - assert_eq!(read_optional(&mut b, rmp::decode::read_u64).unwrap(), None); + assert_eq!(decode::>(&mut b).unwrap(), None); assert!(b.remaining_slice().is_empty()); } #[test] - fn read_optional_string_via_closure() { + fn decode_optional_string() { let buf = enc(|v| rmp::encode::write_str(v, "x").unwrap()); let mut b = Bytes::new(&buf); assert_eq!( - read_optional(&mut b, read_string).unwrap(), + decode::>(&mut b).unwrap(), Some("x".to_string()) ); } #[test] - fn read_optional_forwards_non_nil_errors() { + fn decode_optional_forwards_non_nil_errors() { // A bool where a u64 is expected is a type mismatch that is NOT nil. let buf = enc(|v| rmp::encode::write_bool(v, true).unwrap()); let mut b = Bytes::new(&buf); - assert!(read_optional(&mut b, rmp::decode::read_u64).is_err()); + assert!(decode::>(&mut b).is_err()); } fn array_of(len: u32) -> Vec { @@ -353,17 +405,17 @@ mod tests { } #[test] - fn read_array_len_returns_count_for_manual_range_checks() { + fn decode_array_len_returns_count_for_manual_range_checks() { let buf = array_of(9); let mut b = Bytes::new(&buf); - assert_eq!(read_array_len(&mut b).unwrap(), 9); + assert_eq!(decode_array_len(&mut b).unwrap(), 9); } #[test] fn expect_eof_ok_when_consumed() { let buf = enc(|v| rmp::encode::write_u8(v, 1).unwrap()); let mut b = Bytes::new(&buf); - read_with(&mut b, rmp::decode::read_u8).unwrap(); + decode::(&mut b).unwrap(); assert!(expect_eof(&b).is_ok()); } @@ -381,10 +433,7 @@ mod tests { let mut out = Vec::new(); write_optional(&mut out, Some(99u64), rmp::encode::write_u64).unwrap(); let mut b = Bytes::new(&out); - assert_eq!( - read_optional(&mut b, rmp::decode::read_u64).unwrap(), - Some(99) - ); + assert_eq!(decode::>(&mut b).unwrap(), Some(99)); } #[test] @@ -392,7 +441,7 @@ mod tests { let mut out = Vec::new(); write_optional::<_, u64>(&mut out, None, rmp::encode::write_u64).unwrap(); let mut b = Bytes::new(&out); - assert_eq!(read_optional(&mut b, rmp::decode::read_u64).unwrap(), None); + assert_eq!(decode::>(&mut b).unwrap(), None); } #[test] @@ -401,13 +450,13 @@ mod tests { write_optional(&mut out, Some("hi"), rmp::encode::write_str).unwrap(); let mut b = Bytes::new(&out); assert_eq!( - read_optional(&mut b, read_string).unwrap(), + decode::>(&mut b).unwrap(), Some("hi".to_string()) ); } #[test] - fn read_optional_nil_string_advances_to_next_field() { + fn decode_optional_nil_string_advances_to_next_field() { // A nil optional-string field followed by a u64. The nil read must consume // exactly the marker so the following field decodes correctly. let mut out = Vec::new(); @@ -415,8 +464,8 @@ mod tests { rmp::encode::write_u64(&mut out, 1234).unwrap(); let mut b = Bytes::new(&out); - assert_eq!(read_optional(&mut b, read_string).unwrap(), None); - assert_eq!(read_with(&mut b, rmp::decode::read_u64).unwrap(), 1234); + assert_eq!(decode::>(&mut b).unwrap(), None); + assert_eq!(decode::(&mut b).unwrap(), 1234); assert!(expect_eof(&b).is_ok()); } @@ -425,31 +474,31 @@ mod tests { proptest! { #![proptest_config(ProptestConfig::with_cases(1024))] - // A string survives an encode -> read_string round trip exactly. + // A string survives an encode -> decode:: round trip exactly. #[test] - fn string_round_trips(s in r"(?s).*") { + fn decode_string_proptest_round_trips(s in r"(?s).*") { let buf = enc(|v| rmp::encode::write_str(v, &s).unwrap()); let mut b = Bytes::new(&buf); - prop_assert_eq!(read_string(&mut b).unwrap(), s); + prop_assert_eq!(decode::(&mut b).unwrap(), s); prop_assert!(b.remaining_slice().is_empty()); } - // Option survives write_optional -> read_optional. + // Option survives write_optional -> decode::>. #[test] - fn optional_u64_round_trips(v in proptest::option::of(any::())) { + fn decode_optional_u64_round_trips(v in proptest::option::of(any::())) { let mut out = Vec::new(); write_optional(&mut out, v, rmp::encode::write_u64).unwrap(); let mut b = Bytes::new(&out); - prop_assert_eq!(read_optional(&mut b, rmp::decode::read_u64).unwrap(), v); + prop_assert_eq!(decode::>(&mut b).unwrap(), v); } - // Option survives the closure-based optional round trip. + // Option survives the optional round trip. #[test] - fn optional_string_round_trips(v in proptest::option::of(r"(?s).*")) { + fn decode_optional_string_round_trips(v in proptest::option::of(r"(?s).*")) { let mut out = Vec::new(); write_optional(&mut out, v.as_deref(), rmp::encode::write_str).unwrap(); let mut b = Bytes::new(&out); - prop_assert_eq!(read_optional(&mut b, read_string).unwrap(), v); + prop_assert_eq!(decode::>(&mut b).unwrap(), v); prop_assert!(b.remaining_slice().is_empty()); } @@ -469,9 +518,9 @@ mod tests { let mut b = Bytes::new(&out); prop_assert_eq!(expect_array_len(&mut b, 3).unwrap(), 3); - prop_assert_eq!(read_string(&mut b).unwrap(), id); - prop_assert_eq!(read_with(&mut b, rmp::decode::read_u64).unwrap(), ts); - prop_assert_eq!(read_optional(&mut b, rmp::decode::read_u64).unwrap(), deleted); + prop_assert_eq!(decode::(&mut b).unwrap(), id); + prop_assert_eq!(decode::(&mut b).unwrap(), ts); + prop_assert_eq!(decode::>(&mut b).unwrap(), deleted); prop_assert!(expect_eof(&b).is_ok()); } @@ -479,11 +528,11 @@ mod tests { #[test] fn reads_never_panic(raw in proptest::collection::vec(any::(), 0..64)) { let mut b = Bytes::new(&raw); - let _ = read_string(&mut b); + let _ = decode::(&mut b); let mut b = Bytes::new(&raw); - let _ = read_array_len(&mut b); + let _ = decode_array_len(&mut b); let mut b = Bytes::new(&raw); - let _ = read_optional(&mut b, rmp::decode::read_u64); + let _ = decode::>(&mut b); } } } diff --git a/crates/atuin-dotfiles/src/shell.rs b/crates/atuin-dotfiles/src/shell.rs index 87ccba93946..708ccd8fc3d 100644 --- a/crates/atuin-dotfiles/src/shell.rs +++ b/crates/atuin-dotfiles/src/shell.rs @@ -1,8 +1,8 @@ use eyre::Result; -use rmp::{decode, encode}; +use rmp::encode; use serde::Serialize; -use atuin_common::rmp::{expect_array_len, expect_eof, read_string, read_with}; +use atuin_common::rmp::{Bytes, decode, expect_array_len, expect_eof}; use atuin_common::shell::{Shell, ShellError}; use crate::store::AliasStore; @@ -42,12 +42,12 @@ impl Var { Ok(()) } - pub fn deserialize(bytes: &mut decode::Bytes) -> Result { + pub fn deserialize(bytes: &mut Bytes) -> Result { expect_array_len(bytes, 3)?; - let name = read_string(bytes)?; - let value = read_string(bytes)?; - let export = read_with(bytes, decode::read_bool)?; + let name = decode::(bytes)?; + let value = decode::(bytes)?; + let export = decode::(bytes)?; expect_eof(bytes)?; diff --git a/crates/atuin-dotfiles/src/store.rs b/crates/atuin-dotfiles/src/store.rs index fac79626132..31e8c8129c8 100644 --- a/crates/atuin-dotfiles/src/store.rs +++ b/crates/atuin-dotfiles/src/store.rs @@ -6,7 +6,7 @@ use atuin_client::record::sqlite_store::SqliteStore; // While we will support a range of shell config, I'd rather have a larger number of small records // + stores, rather than one mega config store. use atuin_common::record::{DecryptedData, Host, HostId}; -use atuin_common::rmp::{expect_array_len, expect_eof, read_string, read_with}; +use atuin_common::rmp::{Bytes, decode, expect_array_len, expect_eof}; use atuin_common::utils::unquote; use eyre::{Result, bail, eyre}; @@ -54,21 +54,19 @@ impl AliasRecord { } pub fn deserialize(data: &DecryptedData, version: &str) -> Result { - use rmp::decode; - match version { CONFIG_SHELL_ALIAS_VERSION => { - let mut bytes = decode::Bytes::new(&data.0); + let mut bytes = Bytes::new(&data.0); - let record_type = read_with(&mut bytes, decode::read_u8)?; + let record_type = decode::(&mut bytes)?; match record_type { // create 0 => { expect_array_len(&mut bytes, 2)?; - let name = read_string(&mut bytes)?; - let value = read_string(&mut bytes)?; + let name = decode::(&mut bytes)?; + let value = decode::(&mut bytes)?; expect_eof(&bytes)?; @@ -79,7 +77,7 @@ impl AliasRecord { 1 => { expect_array_len(&mut bytes, 1)?; - let key = read_string(&mut bytes)?; + let key = decode::(&mut bytes)?; expect_eof(&bytes)?; diff --git a/crates/atuin-dotfiles/src/store/var.rs b/crates/atuin-dotfiles/src/store/var.rs index ec2cc723878..0aec7c3e833 100644 --- a/crates/atuin-dotfiles/src/store/var.rs +++ b/crates/atuin-dotfiles/src/store/var.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; use atuin_client::record::sqlite_store::SqliteStore; use atuin_common::record::{DecryptedData, Host, HostId}; -use atuin_common::rmp::{expect_array_len, expect_eof, read_string, read_with}; +use atuin_common::rmp::{Bytes, decode, expect_array_len, expect_eof}; use eyre::{Result, bail, eyre}; use atuin_client::record::encryption::PASETO_V4; @@ -48,13 +48,11 @@ impl VarRecord { } pub fn deserialize(data: &DecryptedData, version: &str) -> Result { - use rmp::decode; - match version { DOTFILES_VAR_VERSION => { - let mut bytes = decode::Bytes::new(&data.0); + let mut bytes = Bytes::new(&data.0); - let record_type = read_with(&mut bytes, decode::read_u8)?; + let record_type = decode::(&mut bytes)?; match record_type { // create @@ -67,7 +65,7 @@ impl VarRecord { 1 => { expect_array_len(&mut bytes, 1)?; - let key = read_string(&mut bytes)?; + let key = decode::(&mut bytes)?; expect_eof(&bytes)?; diff --git a/crates/atuin-kv/src/store/record.rs b/crates/atuin-kv/src/store/record.rs index b48d2ddf749..a39f2e8870a 100644 --- a/crates/atuin-kv/src/store/record.rs +++ b/crates/atuin-kv/src/store/record.rs @@ -1,5 +1,5 @@ use atuin_common::record::DecryptedData; -use atuin_common::rmp::{expect_array_len, expect_eof, read_string, read_with}; +use atuin_common::rmp::{Bytes, decode, expect_array_len, expect_eof}; use eyre::{Result, bail}; use typed_builder::TypedBuilder; @@ -35,17 +35,15 @@ impl KvRecord { } pub fn deserialize(data: &DecryptedData, version: &str) -> Result { - use rmp::decode; - match version { "v0" => { - let mut bytes = decode::Bytes::new(&data.0); + let mut bytes = Bytes::new(&data.0); expect_array_len(&mut bytes, 3)?; - let namespace = read_string(&mut bytes)?; - let key = read_string(&mut bytes)?; - let value = read_string(&mut bytes)?; + let namespace = decode::(&mut bytes)?; + let key = decode::(&mut bytes)?; + let value = decode::(&mut bytes)?; expect_eof(&bytes)?; @@ -56,16 +54,16 @@ impl KvRecord { }) } KV_VERSION => { - let mut bytes = decode::Bytes::new(&data.0); + let mut bytes = Bytes::new(&data.0); expect_array_len(&mut bytes, 4)?; - let namespace = read_string(&mut bytes)?; - let key = read_string(&mut bytes)?; - let has_value = read_with(&mut bytes, decode::read_bool)?; + let namespace = decode::(&mut bytes)?; + let key = decode::(&mut bytes)?; + let has_value = decode::(&mut bytes)?; let value = if has_value { - Some(read_string(&mut bytes)?) + Some(decode::(&mut bytes)?) } else { None }; diff --git a/crates/atuin-scripts/src/store/record.rs b/crates/atuin-scripts/src/store/record.rs index 63e35165385..760a0c7dd9e 100644 --- a/crates/atuin-scripts/src/store/record.rs +++ b/crates/atuin-scripts/src/store/record.rs @@ -47,34 +47,33 @@ impl ScriptRecord { } pub fn deserialize(data: &DecryptedData, version: &str) -> Result { - use atuin_common::rmp::{read_string, read_with}; - use rmp::decode; + use atuin_common::rmp::{Bytes, decode, decode_bin_len}; match version { SCRIPT_VERSION => { - let mut bytes = decode::Bytes::new(&data.0); + let mut bytes = Bytes::new(&data.0); - let record_type = read_with(&mut bytes, decode::read_u8)?; + let record_type = decode::(&mut bytes)?; match record_type { // create 0 => { // written by encode::write_bin above - let _ = read_with(&mut bytes, decode::read_bin_len)?; + let _ = decode_bin_len(&mut bytes)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Create(script)) } // delete 1 => { - let id = read_string(&mut bytes)?; + let id = decode::(&mut bytes)?; Ok(ScriptRecord::Delete(Uuid::parse_str(&id)?)) } // update 2 => { // written by encode::write_bin above - let _ = read_with(&mut bytes, decode::read_bin_len)?; + let _ = decode_bin_len(&mut bytes)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Update(script)) } diff --git a/crates/atuin-scripts/src/store/script.rs b/crates/atuin-scripts/src/store/script.rs index fdf13b9c5ff..ce5b078e06f 100644 --- a/crates/atuin-scripts/src/store/script.rs +++ b/crates/atuin-scripts/src/store/script.rs @@ -2,7 +2,7 @@ use atuin_common::record::DecryptedData; use eyre::Result; use uuid::Uuid; -use rmp::{decode, encode}; +use rmp::encode; use typed_builder::TypedBuilder; pub const SCRIPT_VERSION: &str = "v0"; @@ -60,26 +60,26 @@ impl Script { } pub fn deserialize(bytes: &[u8]) -> Result { - use atuin_common::rmp::{expect_array_len, expect_eof, read_array_len, read_string}; + use atuin_common::rmp::{Bytes, decode, decode_array_len, expect_array_len, expect_eof}; - let mut bytes = decode::Bytes::new(bytes); + let mut bytes = Bytes::new(bytes); expect_array_len(&mut bytes, 6)?; - let id = read_string(&mut bytes)?; - let name = read_string(&mut bytes)?; - let description = read_string(&mut bytes)?; - let shebang = read_string(&mut bytes)?; + let id = decode::(&mut bytes)?; + let name = decode::(&mut bytes)?; + let description = decode::(&mut bytes)?; + let shebang = decode::(&mut bytes)?; - let tags_len = read_array_len(&mut bytes)?; + let tags_len = decode_array_len(&mut bytes)?; let mut tags = Vec::new(); for _ in 0..tags_len { - let tag = read_string(&mut bytes)?; + let tag = decode::(&mut bytes)?; tags.push(tag); } - let script = read_string(&mut bytes)?; + let script = decode::(&mut bytes)?; expect_eof(&bytes)?; From a35d5847582ddee12964b80fa8287ade8225703c Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 18 Jul 2026 23:50:05 -0700 Subject: [PATCH 19/27] refactor(common): split rmp into decode/encode facade modules --- crates/atuin-client/src/encryption.rs | 14 +- crates/atuin-client/src/history.rs | 38 +- crates/atuin-client/src/history/store.rs | 12 +- crates/atuin-common/src/rmp.rs | 538 ----------------------- crates/atuin-common/src/rmp/decode.rs | 475 ++++++++++++++++++++ crates/atuin-common/src/rmp/encode.rs | 79 ++++ crates/atuin-common/src/rmp/mod.rs | 10 + crates/atuin-dotfiles/src/shell.rs | 14 +- crates/atuin-dotfiles/src/store.rs | 20 +- crates/atuin-dotfiles/src/store/var.rs | 12 +- crates/atuin-kv/src/store/record.rs | 26 +- crates/atuin-scripts/src/store/record.rs | 12 +- crates/atuin-scripts/src/store/script.rs | 22 +- 13 files changed, 649 insertions(+), 623 deletions(-) delete mode 100644 crates/atuin-common/src/rmp.rs create mode 100644 crates/atuin-common/src/rmp/decode.rs create mode 100644 crates/atuin-common/src/rmp/encode.rs create mode 100644 crates/atuin-common/src/rmp/mod.rs diff --git a/crates/atuin-client/src/encryption.rs b/crates/atuin-client/src/encryption.rs index d973caee165..c9ead0ca8ed 100644 --- a/crates/atuin-client/src/encryption.rs +++ b/crates/atuin-client/src/encryption.rs @@ -17,7 +17,8 @@ use eyre::{Context, Result, bail, ensure}; use fs_err as fs; use rmp::Marker; -use atuin_common::rmp::{Bytes, decode, decode_bin_len, expect_array_len}; +use atuin_common::rmp::decode::{self, Bytes}; +use atuin_common::rmp::encode; use crate::settings::Settings; @@ -59,11 +60,10 @@ pub fn load_key(settings: &Settings) -> Result { pub fn encode_key(key: &Key) -> Result { let mut buf = vec![]; - rmp::encode::write_array_len(&mut buf, key.len() as u32) + encode::write_array_len(&mut buf, key.len() as u32) .wrap_err("could not encode key to message pack")?; for b in key { - rmp::encode::write_uint(&mut buf, *b as u64) - .wrap_err("could not encode key to message pack")?; + encode::write_uint(&mut buf, *b as u64).wrap_err("could not encode key to message pack")?; } let buf = BASE64_STANDARD.encode(buf); @@ -84,18 +84,18 @@ pub fn decode_key(key: String) -> Result { match Marker::from_u8(buf[0]) { Marker::Bin8 => { - let len = decode_bin_len(&mut bytes)?; + let len = decode::read_bin_len(&mut bytes)?; ensure!(len == 32, "encryption key is not the correct size"); let key = <[u8; 32]>::try_from(bytes.remaining_slice()) .context("could not decode encryption key")?; Ok(key.into()) } Marker::Array16 => { - expect_array_len(&mut bytes, 32)?; + decode::expect_array_len(&mut bytes, 32)?; let mut key = Key::default(); for i in &mut key { - *i = decode::(&mut bytes)?; + *i = decode::read_u8(&mut bytes)?; } Ok(key) } diff --git a/crates/atuin-client/src/history.rs b/crates/atuin-client/src/history.rs index 44da29a682d..f2d7b124775 100644 --- a/crates/atuin-client/src/history.rs +++ b/crates/atuin-client/src/history.rs @@ -1,4 +1,3 @@ -use rmp::encode; use std::env; use atuin_common::record::DecryptedData; @@ -9,7 +8,8 @@ use eyre::{Result, bail}; use crate::secrets::SECRET_PATTERNS_RE; use crate::settings::Settings; use crate::utils::get_host_user; -use atuin_common::rmp::{Bytes, EncodeError, decode, decode_array_len, write_optional}; +use atuin_common::rmp::decode::{self, Bytes}; +use atuin_common::rmp::encode::{self, EncodeError}; use time::OffsetDateTime; pub(crate) mod builder; @@ -229,14 +229,14 @@ impl History { encode::write_str(&mut output, &self.session)?; encode::write_str(&mut output, &self.hostname)?; - write_optional( + encode::write_optional( &mut output, self.deleted_at.map(|d| d.unix_timestamp_nanos() as u64), encode::write_u64, )?; encode::write_str(&mut output, self.author.as_str())?; - write_optional(&mut output, self.intent.as_deref(), encode::write_str)?; - write_optional(&mut output, self.shell.as_deref(), encode::write_str)?; + encode::write_optional(&mut output, self.intent.as_deref(), encode::write_str)?; + encode::write_optional(&mut output, self.shell.as_deref(), encode::write_str)?; Ok(DecryptedData(output)) } @@ -247,30 +247,30 @@ impl History { let mut bytes = Bytes::new(bytes); - let real_version = decode::(&mut bytes)?; + let real_version = decode::read_u16(&mut bytes)?; if real_version != version.as_int() { bail!("expected to decode {version} record, found v{real_version}"); } - let nfields = decode_array_len(&mut bytes)?; + let nfields = decode::read_array_len(&mut bytes)?; let min_fields = version.min_fields(); if nfields < min_fields || version.max_fields().is_some_and(|max| nfields > max) { bail!("unexpected number of fields ({nfields}) for history version {version}"); } - let id = decode::(&mut bytes)?; - let timestamp = decode::(&mut bytes)?; - let duration = decode::(&mut bytes)?; - let exit = decode::(&mut bytes)?; + let id = decode::read_string(&mut bytes)?; + let timestamp = decode::read_u64(&mut bytes)?; + let duration = decode::read_i64(&mut bytes)?; + let exit = decode::read_i64(&mut bytes)?; - let command = decode::(&mut bytes)?; - let cwd = decode::(&mut bytes)?; - let session = decode::(&mut bytes)?; - let hostname = decode::(&mut bytes)?; - let deleted_at = decode::>(&mut bytes)?; + let command = decode::read_string(&mut bytes)?; + let cwd = decode::read_string(&mut bytes)?; + let session = decode::read_string(&mut bytes)?; + let hostname = decode::read_string(&mut bytes)?; + let deleted_at = decode::read_optional(&mut bytes, decode::read_u64)?; let author = if version >= Version::One { - decode::>(&mut bytes)? + decode::read_optional(&mut bytes, decode::read_string)? } else { None }; @@ -280,13 +280,13 @@ impl History { Version::One => nfields > min_fields, Version::Two => true, } { - decode::>(&mut bytes)? + decode::read_optional(&mut bytes, decode::read_string)? } else { None }; let shell = if version >= Version::Two { - decode::>(&mut bytes)? + decode::read_optional(&mut bytes, decode::read_string)? } else { None }; diff --git a/crates/atuin-client/src/history/store.rs b/crates/atuin-client/src/history/store.rs index 883defbf796..7deab24583d 100644 --- a/crates/atuin-client/src/history/store.rs +++ b/crates/atuin-client/src/history/store.rs @@ -9,7 +9,7 @@ use crate::{ record::{encryption::PASETO_V4, sqlite_store::SqliteStore, store::Store}, }; use atuin_common::record::{DecryptedData, Host, HostId, Record, RecordId, RecordIdx}; -use atuin_common::rmp::{Bytes, decode, decode_bin_len, expect_eof}; +use atuin_common::rmp::decode::{self, Bytes}; use super::{HISTORY_TAG, History, HistoryId, Version}; @@ -47,7 +47,7 @@ impl HistoryRecord { pub fn serialize(&self) -> Result { // probably don't actually need to use rmp here, but if we ever need to extend it, it's a // nice wrapper around raw byte stuff - use rmp::encode; + use atuin_common::rmp::encode; let mut output = vec![]; @@ -73,14 +73,14 @@ impl HistoryRecord { pub fn deserialize(bytes: &DecryptedData, version: &str) -> Result { let mut bytes = Bytes::new(&bytes.0); - let record_type = decode::(&mut bytes)?; + let record_type = decode::read_u8(&mut bytes)?; match record_type { // 0 -> HistoryRecord::Create 0 => { // not super useful to us atm, but perhaps in the future // written by write_bin above - let _ = decode_bin_len(&mut bytes)?; + let _ = decode::read_bin_len(&mut bytes)?; let record = History::deserialize(bytes.remaining_slice(), version)?; @@ -89,8 +89,8 @@ impl HistoryRecord { // 1 -> HistoryRecord::Delete 1 => { - let id = decode::(&mut bytes)?; - expect_eof(&bytes)?; + let id = decode::read_string(&mut bytes)?; + decode::expect_eof(&bytes)?; Ok(HistoryRecord::Delete(id.into())) } diff --git a/crates/atuin-common/src/rmp.rs b/crates/atuin-common/src/rmp.rs deleted file mode 100644 index 4b698037640..00000000000 --- a/crates/atuin-common/src/rmp.rs +++ /dev/null @@ -1,538 +0,0 @@ -//! MessagePack encode/decode helpers built on [`rmp`]. -//! -//! `rmp`'s own error types are awkward: some don't implement [`Display`] for -//! every `E`, none say which variant they are, and the decode cursor offers no -//! owned-string read, no nil-aware optional read, and no structural checks -//! (array length, end-of-input). This module fills those gaps. -//! -//! Values are read through the generic [`decode()`] function, backed by the -//! [`Decode`] trait: `decode::(&mut bytes)?`, -//! `decode::>(&mut bytes)?`. Structural markers that aren't values -//! have their own functions — [`decode_array_len`], [`decode_bin_len`], -//! [`expect_array_len`] and [`expect_eof`] — and [`write_optional`] handles the -//! encode side. All of these return [`DecodeError`] / [`EncodeError`], which -//! carry legible messages and convert cleanly into [`eyre::Report`]. -//! -//! Because every decode read returns [`DecodeError`], a decoder can use `?` -//! throughout and convert once at the boundary, rather than hand-rolling a -//! `map_err` at every read. -//! -//! [`Display`]: std::fmt::Display - -use rmp::Marker; -use rmp::decode::bytes::BytesReadError; -use rmp::decode::{ - self, DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError, -}; -use rmp::encode::{self, RmpWrite, RmpWriteErr, ValueWriteError}; - -/// Re-exported so decoders need only depend on this module, not `rmp::decode`. -pub use rmp::decode::bytes::Bytes; - -/// An error encountered while encoding a MessagePack value. -/// -/// Wraps [`ValueWriteError`] with a message that names the failing write and its -/// inner I/O error — neither of which `rmp`'s own [`Display`] reports. Implements -/// [`std::error::Error`], so it converts into [`eyre::Report`] with `?`. -/// -/// [`Display`]: std::fmt::Display -#[derive(Debug, derive_more::Display, derive_more::From, thiserror::Error)] -#[display("could not write MessagePack value: {_0:?}")] -pub struct EncodeError(ValueWriteError); - -/// An error encountered while decoding a MessagePack value. -/// -/// Wraps the three error types `rmp`'s decode functions return, and adds two -/// structural variants for checks `rmp` does not perform itself: -/// [`UnexpectedArrayLen`](Self::UnexpectedArrayLen) and -/// [`TrailingBytes`](Self::TrailingBytes). -/// -/// Converts into [`eyre::Report`] via a manual `From` rather than a -/// [`std::error::Error`] impl, because a [`DecodeError`] is not, in general, -/// `'static`. -/// -/// [`Display`]: std::fmt::Display -#[derive(Debug, derive_more::Display)] -pub enum DecodeError<'a, E: RmpReadErr = BytesReadError> { - /// The next value was not a valid UTF-8 string. - #[display("could not decode MessagePack string: {_0:?}")] - DecodeString(DecodeStringError<'a, E>), - /// The next value was not the expected number. - #[display("could not decode MessagePack number: {_0:?}")] - NumValueRead(NumValueReadError), - /// The next value could not be decoded. - #[display("could not decode MessagePack value: {_0:?}")] - ValueRead(ValueReadError), - /// An array marker did not have the expected length. - #[display("expected a MessagePack array of length {expected}, found {actual}")] - UnexpectedArrayLen { expected: u32, actual: u32 }, - /// Input remained after a value was fully decoded. - #[display("{remaining} trailing byte(s) after decoding MessagePack value")] - TrailingBytes { remaining: usize }, -} - -impl<'a, E: RmpReadErr> From> for DecodeError<'a, E> { - fn from(e: DecodeStringError<'a, E>) -> Self { - Self::DecodeString(e) - } -} - -impl From> for DecodeError<'_, E> { - fn from(e: NumValueReadError) -> Self { - Self::NumValueRead(e) - } -} - -impl From> for DecodeError<'_, E> { - fn from(e: ValueReadError) -> Self { - Self::ValueRead(e) - } -} - -impl DecodeError<'_, E> { - /// If this is a type mismatch, the [`Marker`] found instead of the expected - /// type. The [`Option`] implementation of [`Decode`] uses this to recognise - /// nil. - pub fn type_mismatch(&self) -> Option { - match self { - Self::DecodeString(DecodeStringError::TypeMismatch(m)) - | Self::NumValueRead(NumValueReadError::TypeMismatch(m)) - | Self::ValueRead(ValueReadError::TypeMismatch(m)) => Some(*m), - _ => None, - } - } -} - -impl From> for eyre::Report { - fn from(e: DecodeError<'_, E>) -> Self { - eyre::eyre!("{e}") - } -} - -/// A value decodable from a MessagePack [`Bytes`] cursor. -/// -/// Implemented for the scalars atuin's record formats use and for [`Option`] -/// (nil ⇒ [`None`]). Call it through the generic [`decode()`] function; structural -/// markers that aren't values have their own functions ([`decode_array_len`], -/// [`decode_bin_len`]). -pub trait Decode: Sized { - fn decode<'a>(bytes: &mut Bytes<'a>) -> Result>; -} - -/// Decode a [`Decode`] value: `decode::(&mut bytes)?`, -/// `decode::>(&mut bytes)?`. -pub fn decode<'a, T: Decode>(bytes: &mut Bytes<'a>) -> Result> { - T::decode(bytes) -} - -impl Decode for String { - fn decode<'a>(bytes: &mut Bytes<'a>) -> Result> { - let slice = bytes.remaining_slice(); - let (string, rest) = match decode::read_str_from_slice(slice) { - Ok(pair) => pair, - Err(e) => { - if let DecodeStringError::TypeMismatch(_) = e { - // rmp consumes the marker byte on a type mismatch; match that - // so `Option::decode` can detect nil. - bytes - .read_u8() - .expect("TypeMismatch implies the stream contains a marker byte"); - } - return Err(e.into()); - } - }; - *bytes = Bytes::new(rest); - Ok(string.into()) - } -} - -impl Decode for bool { - fn decode<'a>(bytes: &mut Bytes<'a>) -> Result> { - decode::read_bool(bytes).map_err(Into::into) - } -} - -macro_rules! impl_decode_int { - ($($t:ty),+ $(,)?) => {$( - impl Decode for $t { - fn decode<'a>(bytes: &mut Bytes<'a>) -> Result> { - decode::read_int(bytes).map_err(Into::into) - } - } - )+}; -} -impl_decode_int!(u8, u16, u64, i64); - -impl Decode for Option { - /// Decodes a `T`, mapping a nil marker to [`None`]. - fn decode<'a>(bytes: &mut Bytes<'a>) -> Result> { - match T::decode(bytes) { - Ok(v) => Ok(Some(v)), - Err(e) => { - if let Some(Marker::Null) = e.type_mismatch() { - Ok(None) - } else { - Err(e) - } - } - } - } -} - -/// Decode an array-length marker (the header count, not a value). -pub fn decode_array_len<'a>(bytes: &mut Bytes<'a>) -> Result> { - decode::read_array_len(bytes).map_err(Into::into) -} - -/// Decode a binary-length marker (the header, before the raw payload bytes). -pub fn decode_bin_len<'a>(bytes: &mut Bytes<'a>) -> Result> { - decode::read_bin_len(bytes).map_err(Into::into) -} - -/// Decode an array-length marker and require it to equal `expected`, else -/// [`DecodeError::UnexpectedArrayLen`]. For a forward-compatible field count, -/// use [`decode_array_len`] and range-check yourself. -pub fn expect_array_len<'a>(bytes: &mut Bytes<'a>, expected: u32) -> Result> { - let actual = decode_array_len(bytes)?; - if actual == expected { - Ok(actual) - } else { - Err(DecodeError::UnexpectedArrayLen { expected, actual }) - } -} - -/// Succeed only if the cursor is at end-of-input, else [`DecodeError::TrailingBytes`]. -pub fn expect_eof<'a>(bytes: &Bytes<'a>) -> Result<(), DecodeError<'a>> { - let remaining = bytes.remaining_slice().len(); - if remaining == 0 { - Ok(()) - } else { - Err(DecodeError::TrailingBytes { remaining }) - } -} - -/// Write an optional value, encoding [`None`] as [`Marker::Null`]. -pub fn write_optional( - out: &mut W, - value: Option, - write: impl FnOnce(&mut W, T) -> Result<(), ValueWriteError>, -) -> Result<(), EncodeError> { - match value { - Some(v) => write(out, v).map_err(EncodeError::from), - None => encode::write_nil(out) - .map_err(|e| EncodeError::from(ValueWriteError::InvalidMarkerWrite(e))), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - use rmp::decode::bytes::Bytes; - use rstest::rstest; - - // Build a real DecodeError by asking rmp to read the wrong type. - fn type_mismatch_error<'a>() -> DecodeError<'a> { - // 0xc0 is the nil marker; reading it as a u64 is a type mismatch. - let mut bytes = Bytes::new(&[0xc0]); - rmp::decode::read_u64(&mut bytes) - .map_err(DecodeError::from) - .unwrap_err() - } - - #[test] - fn type_mismatch_exposes_marker() { - assert_eq!( - type_mismatch_error().type_mismatch(), - Some(rmp::Marker::Null) - ); - } - - #[rstest] - #[case::array_len( - DecodeError::<'static>::UnexpectedArrayLen { expected: 3, actual: 5 }, - None - )] - #[case::trailing(DecodeError::<'static>::TrailingBytes { remaining: 2 }, None)] - fn structural_variants_have_no_marker( - #[case] err: DecodeError<'static>, - #[case] expected: Option, - ) { - assert_eq!(err.type_mismatch(), expected); - } - - #[test] - fn decode_error_converts_to_eyre_with_message() { - let report: eyre::Report = - DecodeError::<'static, BytesReadError>::TrailingBytes { remaining: 4 }.into(); - assert!(report.to_string().contains("trailing")); - } - - #[test] - fn display_messages_are_legible() { - assert_eq!( - DecodeError::<'static, BytesReadError>::UnexpectedArrayLen { - expected: 3, - actual: 5 - } - .to_string(), - "expected a MessagePack array of length 3, found 5", - ); - } - - // Encode helpers used only by tests, to build inputs. - fn enc)>(f: F) -> Vec { - let mut v = Vec::new(); - f(&mut v); - v - } - - #[test] - fn decode_string_round_trips() { - let buf = enc(|v| rmp::encode::write_str(v, "héllo 🦀").unwrap()); - let mut bytes = Bytes::new(&buf); - assert_eq!(decode::(&mut bytes).unwrap(), "héllo 🦀"); - assert!(bytes.remaining_slice().is_empty()); - } - - #[test] - fn decode_string_on_wrong_type_errors_and_consumes_marker() { - // A lone nil marker: decode:: must fail *and* consume the marker - // so a following decode::> can observe end-of-input correctly. - let mut bytes = Bytes::new(&[0xc0]); - assert!(decode::(&mut bytes).is_err()); - assert!( - bytes.remaining_slice().is_empty(), - "marker byte must be consumed" - ); - } - - #[test] - fn decode_converts_rmp_errors() { - let buf = enc(|v| rmp::encode::write_u64(v, 42).unwrap()); - let mut bytes = Bytes::new(&buf); - assert_eq!(decode::(&mut bytes).unwrap(), 42); - } - - #[rstest] - #[case::bool_true(enc(|v| rmp::encode::write_bool(v, true).unwrap()), true)] - #[case::bool_false(enc(|v| rmp::encode::write_bool(v, false).unwrap()), false)] - fn decode_bool_round_trips(#[case] buf: Vec, #[case] expected: bool) { - let mut b = Bytes::new(&buf); - assert_eq!(decode::(&mut b).unwrap(), expected); - assert!(b.remaining_slice().is_empty()); - } - - #[test] - fn decode_u8_round_trips() { - let buf = enc(|v| rmp::encode::write_u8(v, 200).unwrap()); - let mut b = Bytes::new(&buf); - assert_eq!(decode::(&mut b).unwrap(), 200); - assert!(b.remaining_slice().is_empty()); - } - - #[test] - fn decode_u16_round_trips() { - let buf = enc(|v| rmp::encode::write_u16(v, 40000).unwrap()); - let mut b = Bytes::new(&buf); - assert_eq!(decode::(&mut b).unwrap(), 40000); - assert!(b.remaining_slice().is_empty()); - } - - #[test] - fn decode_i64_round_trips() { - let buf = enc(|v| { - rmp::encode::write_sint(v, -123456789).unwrap(); - }); - let mut b = Bytes::new(&buf); - assert_eq!(decode::(&mut b).unwrap(), -123456789); - assert!(b.remaining_slice().is_empty()); - } - - #[test] - fn decode_optional_some_and_none() { - let some = enc(|v| rmp::encode::write_u64(v, 7).unwrap()); - let mut b = Bytes::new(&some); - assert_eq!(decode::>(&mut b).unwrap(), Some(7)); - - let none = enc(|v| rmp::encode::write_nil(v).unwrap()); - let mut b = Bytes::new(&none); - assert_eq!(decode::>(&mut b).unwrap(), None); - assert!(b.remaining_slice().is_empty()); - } - - #[test] - fn decode_optional_string() { - let buf = enc(|v| rmp::encode::write_str(v, "x").unwrap()); - let mut b = Bytes::new(&buf); - assert_eq!( - decode::>(&mut b).unwrap(), - Some("x".to_string()) - ); - } - - #[test] - fn decode_optional_forwards_non_nil_errors() { - // A bool where a u64 is expected is a type mismatch that is NOT nil. - let buf = enc(|v| rmp::encode::write_bool(v, true).unwrap()); - let mut b = Bytes::new(&buf); - assert!(decode::>(&mut b).is_err()); - } - - fn array_of(len: u32) -> Vec { - enc(|v| { - rmp::encode::write_array_len(v, len).unwrap(); - }) - } - - #[test] - fn expect_array_len_exact_ok() { - let buf = array_of(3); - let mut b = Bytes::new(&buf); - assert_eq!(expect_array_len(&mut b, 3).unwrap(), 3); - } - - #[test] - fn expect_array_len_mismatch_reports_expected_and_actual() { - let buf = array_of(5); - let mut b = Bytes::new(&buf); - match expect_array_len(&mut b, 3) { - Err(DecodeError::UnexpectedArrayLen { expected, actual }) => { - assert_eq!((expected, actual), (3, 5)); - } - other => panic!("expected UnexpectedArrayLen, got {other:?}"), - } - } - - #[test] - fn decode_array_len_returns_count_for_manual_range_checks() { - let buf = array_of(9); - let mut b = Bytes::new(&buf); - assert_eq!(decode_array_len(&mut b).unwrap(), 9); - } - - #[test] - fn expect_eof_ok_when_consumed() { - let buf = enc(|v| rmp::encode::write_u8(v, 1).unwrap()); - let mut b = Bytes::new(&buf); - decode::(&mut b).unwrap(); - assert!(expect_eof(&b).is_ok()); - } - - #[test] - fn expect_eof_reports_remaining() { - let b = Bytes::new(&[0x01, 0x02, 0x03]); - match expect_eof(&b) { - Err(DecodeError::TrailingBytes { remaining }) => assert_eq!(remaining, 3), - other => panic!("expected TrailingBytes, got {other:?}"), - } - } - - #[test] - fn write_optional_some_then_read_back() { - let mut out = Vec::new(); - write_optional(&mut out, Some(99u64), rmp::encode::write_u64).unwrap(); - let mut b = Bytes::new(&out); - assert_eq!(decode::>(&mut b).unwrap(), Some(99)); - } - - #[test] - fn write_optional_none_writes_nil() { - let mut out = Vec::new(); - write_optional::<_, u64>(&mut out, None, rmp::encode::write_u64).unwrap(); - let mut b = Bytes::new(&out); - assert_eq!(decode::>(&mut b).unwrap(), None); - } - - #[test] - fn write_optional_str() { - let mut out = Vec::new(); - write_optional(&mut out, Some("hi"), rmp::encode::write_str).unwrap(); - let mut b = Bytes::new(&out); - assert_eq!( - decode::>(&mut b).unwrap(), - Some("hi".to_string()) - ); - } - - #[test] - fn decode_optional_nil_string_advances_to_next_field() { - // A nil optional-string field followed by a u64. The nil read must consume - // exactly the marker so the following field decodes correctly. - let mut out = Vec::new(); - write_optional::<_, &str>(&mut out, None, rmp::encode::write_str).unwrap(); - rmp::encode::write_u64(&mut out, 1234).unwrap(); - - let mut b = Bytes::new(&out); - assert_eq!(decode::>(&mut b).unwrap(), None); - assert_eq!(decode::(&mut b).unwrap(), 1234); - assert!(expect_eof(&b).is_ok()); - } - - use proptest::prelude::*; - - proptest! { - #![proptest_config(ProptestConfig::with_cases(1024))] - - // A string survives an encode -> decode:: round trip exactly. - #[test] - fn decode_string_proptest_round_trips(s in r"(?s).*") { - let buf = enc(|v| rmp::encode::write_str(v, &s).unwrap()); - let mut b = Bytes::new(&buf); - prop_assert_eq!(decode::(&mut b).unwrap(), s); - prop_assert!(b.remaining_slice().is_empty()); - } - - // Option survives write_optional -> decode::>. - #[test] - fn decode_optional_u64_round_trips(v in proptest::option::of(any::())) { - let mut out = Vec::new(); - write_optional(&mut out, v, rmp::encode::write_u64).unwrap(); - let mut b = Bytes::new(&out); - prop_assert_eq!(decode::>(&mut b).unwrap(), v); - } - - // Option survives the optional round trip. - #[test] - fn decode_optional_string_round_trips(v in proptest::option::of(r"(?s).*")) { - let mut out = Vec::new(); - write_optional(&mut out, v.as_deref(), rmp::encode::write_str).unwrap(); - let mut b = Bytes::new(&out); - prop_assert_eq!(decode::>(&mut b).unwrap(), v); - prop_assert!(b.remaining_slice().is_empty()); - } - - // A full array record (len + fields + optional tail) round trips, and the - // cursor is exactly exhausted afterwards. - #[test] - fn record_round_trips( - id in r"[a-z]{0,16}", - ts in any::(), - deleted in proptest::option::of(any::()), - ) { - let mut out = Vec::new(); - rmp::encode::write_array_len(&mut out, 3).unwrap(); - rmp::encode::write_str(&mut out, &id).unwrap(); - rmp::encode::write_u64(&mut out, ts).unwrap(); - write_optional(&mut out, deleted, rmp::encode::write_u64).unwrap(); - - let mut b = Bytes::new(&out); - prop_assert_eq!(expect_array_len(&mut b, 3).unwrap(), 3); - prop_assert_eq!(decode::(&mut b).unwrap(), id); - prop_assert_eq!(decode::(&mut b).unwrap(), ts); - prop_assert_eq!(decode::>(&mut b).unwrap(), deleted); - prop_assert!(expect_eof(&b).is_ok()); - } - - // Reads never panic on arbitrary bytes — they return Err instead. - #[test] - fn reads_never_panic(raw in proptest::collection::vec(any::(), 0..64)) { - let mut b = Bytes::new(&raw); - let _ = decode::(&mut b); - let mut b = Bytes::new(&raw); - let _ = decode_array_len(&mut b); - let mut b = Bytes::new(&raw); - let _ = decode::>(&mut b); - } - } -} diff --git a/crates/atuin-common/src/rmp/decode.rs b/crates/atuin-common/src/rmp/decode.rs new file mode 100644 index 00000000000..ab39ee91f66 --- /dev/null +++ b/crates/atuin-common/src/rmp/decode.rs @@ -0,0 +1,475 @@ +//! MessagePack decode helpers built on [`rmp::decode`]. +//! +//! Mirrors `rmp::decode`'s `read_*` shape, but every read returns our own +//! [`DecodeError`] (which converts cleanly into [`eyre::Report`]) so a decoder +//! can use `?` throughout and convert once at the boundary. Adds an owned +//! [`read_string`], a nil-aware [`read_optional`], and the structural checks +//! [`read_array_len`]/[`expect_array_len`] and [`expect_eof`] that `rmp` does +//! not perform itself. + +use rmp::Marker; +use rmp::decode as mp; +use rmp::decode::bytes::BytesReadError; +use rmp::decode::{DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError}; + +/// Re-exported so decoders need only depend on this module, not `rmp::decode`. +pub use rmp::decode::bytes::Bytes; + +/// An error encountered while decoding a MessagePack value. +/// +/// Wraps the three error types `rmp`'s decode functions return, and adds two +/// structural variants for checks `rmp` does not perform itself: +/// [`UnexpectedArrayLen`](Self::UnexpectedArrayLen) and +/// [`TrailingBytes`](Self::TrailingBytes). +/// +/// Converts into [`eyre::Report`] via a manual `From` rather than a +/// [`std::error::Error`] impl, because a [`DecodeError`] is not, in general, +/// `'static`. +/// +/// [`Display`]: std::fmt::Display +#[derive(Debug, derive_more::Display)] +pub enum DecodeError<'a, E: RmpReadErr = BytesReadError> { + /// The next value was not a valid UTF-8 string. + #[display("could not decode MessagePack string: {_0:?}")] + DecodeString(DecodeStringError<'a, E>), + /// The next value was not the expected number. + #[display("could not decode MessagePack number: {_0:?}")] + NumValueRead(NumValueReadError), + /// The next value could not be decoded. + #[display("could not decode MessagePack value: {_0:?}")] + ValueRead(ValueReadError), + /// An array marker did not have the expected length. + #[display("expected a MessagePack array of length {expected}, found {actual}")] + UnexpectedArrayLen { expected: u32, actual: u32 }, + /// Input remained after a value was fully decoded. + #[display("{remaining} trailing byte(s) after decoding MessagePack value")] + TrailingBytes { remaining: usize }, +} + +impl<'a, E: RmpReadErr> From> for DecodeError<'a, E> { + fn from(e: DecodeStringError<'a, E>) -> Self { + Self::DecodeString(e) + } +} + +impl From> for DecodeError<'_, E> { + fn from(e: NumValueReadError) -> Self { + Self::NumValueRead(e) + } +} + +impl From> for DecodeError<'_, E> { + fn from(e: ValueReadError) -> Self { + Self::ValueRead(e) + } +} + +impl DecodeError<'_, E> { + /// If this is a type mismatch, the [`Marker`] found instead of the expected + /// type. [`read_optional`] uses this to recognise nil. + pub fn type_mismatch(&self) -> Option { + match self { + Self::DecodeString(DecodeStringError::TypeMismatch(m)) + | Self::NumValueRead(NumValueReadError::TypeMismatch(m)) + | Self::ValueRead(ValueReadError::TypeMismatch(m)) => Some(*m), + _ => None, + } + } +} + +impl From> for eyre::Report { + fn from(e: DecodeError<'_, E>) -> Self { + eyre::eyre!("{e}") + } +} + +macro_rules! read_int { + ($($(#[$meta:meta])* $name:ident -> $t:ty),+ $(,)?) => {$( + $(#[$meta])* + pub fn $name<'a>(bytes: &mut Bytes<'a>) -> Result<$t, DecodeError<'a>> { + mp::read_int(bytes).map_err(Into::into) + } + )+}; +} + +read_int! { + /// Read a `u8` value. Accepts any in-range MessagePack integer encoding. + read_u8 -> u8, + /// Read a `u16` value. Accepts any in-range MessagePack integer encoding. + read_u16 -> u16, + /// Read a `u64` value. Accepts any in-range MessagePack integer encoding. + read_u64 -> u64, + /// Read an `i64` value. Accepts any in-range MessagePack integer encoding. + read_i64 -> i64, +} + +/// Read a `bool`. +pub fn read_bool<'a>(bytes: &mut Bytes<'a>) -> Result> { + mp::read_bool(bytes).map_err(Into::into) +} + +/// Read a binary-blob length header (before the raw payload). +pub fn read_bin_len<'a>(bytes: &mut Bytes<'a>) -> Result> { + mp::read_bin_len(bytes).map_err(Into::into) +} + +/// Read an array-length header. +pub fn read_array_len<'a>(bytes: &mut Bytes<'a>) -> Result> { + mp::read_array_len(bytes).map_err(Into::into) +} + +/// Read an owned [`String`]. +pub fn read_string<'a>(bytes: &mut Bytes<'a>) -> Result> { + let slice = bytes.remaining_slice(); + let (string, rest) = match mp::read_str_from_slice(slice) { + Ok(pair) => pair, + Err(e) => { + if let DecodeStringError::TypeMismatch(_) = e { + // rmp consumes the marker byte on a type mismatch; match that so + // `read_optional` can detect nil. + bytes + .read_u8() + .expect("TypeMismatch implies the stream contains a marker byte"); + } + return Err(e.into()); + } + }; + *bytes = Bytes::new(rest); + Ok(string.into()) +} + +/// Read a value that may be nil, returning [`None`] for nil. `read` decodes the +/// inner value (e.g. [`read_u64`] or [`read_string`]). +pub fn read_optional<'a, T, E>( + bytes: &mut Bytes<'a>, + read: impl FnOnce(&mut Bytes<'a>) -> Result, +) -> Result, DecodeError<'a>> +where + E: Into>, +{ + match read(bytes) { + Ok(v) => Ok(Some(v)), + Err(e) => { + let e = e.into(); + if let Some(Marker::Null) = e.type_mismatch() { + Ok(None) + } else { + Err(e) + } + } + } +} + +/// Read an array-length header and require it to equal `expected`, else +/// [`DecodeError::UnexpectedArrayLen`]. For a forward-compatible field count, +/// use [`read_array_len`] and range-check yourself. +pub fn expect_array_len<'a>(bytes: &mut Bytes<'a>, expected: u32) -> Result> { + let actual = read_array_len(bytes)?; + if actual == expected { + Ok(actual) + } else { + Err(DecodeError::UnexpectedArrayLen { expected, actual }) + } +} + +/// Succeed only if the cursor is at end-of-input, else [`DecodeError::TrailingBytes`]. +pub fn expect_eof<'a>(bytes: &Bytes<'a>) -> Result<(), DecodeError<'a>> { + let remaining = bytes.remaining_slice().len(); + if remaining == 0 { + Ok(()) + } else { + Err(DecodeError::TrailingBytes { remaining }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rmp::encode; + use pretty_assertions::assert_eq; + use rstest::rstest; + + // Build a real DecodeError by asking rmp to read the wrong type. + fn type_mismatch_error<'a>() -> DecodeError<'a> { + // 0xc0 is the nil marker; reading it as a u64 is a type mismatch. + let mut bytes = Bytes::new(&[0xc0]); + read_u64(&mut bytes).unwrap_err() + } + + #[test] + fn type_mismatch_exposes_marker() { + assert_eq!( + type_mismatch_error().type_mismatch(), + Some(rmp::Marker::Null) + ); + } + + #[rstest] + #[case::array_len( + DecodeError::<'static>::UnexpectedArrayLen { expected: 3, actual: 5 }, + None + )] + #[case::trailing(DecodeError::<'static>::TrailingBytes { remaining: 2 }, None)] + fn structural_variants_have_no_marker( + #[case] err: DecodeError<'static>, + #[case] expected: Option, + ) { + assert_eq!(err.type_mismatch(), expected); + } + + #[test] + fn decode_error_converts_to_eyre_with_message() { + let report: eyre::Report = + DecodeError::<'static, BytesReadError>::TrailingBytes { remaining: 4 }.into(); + assert!(report.to_string().contains("trailing")); + } + + #[test] + fn display_messages_are_legible() { + assert_eq!( + DecodeError::<'static, BytesReadError>::UnexpectedArrayLen { + expected: 3, + actual: 5 + } + .to_string(), + "expected a MessagePack array of length 3, found 5", + ); + } + + // Encode helpers used only by tests, to build inputs. + fn enc)>(f: F) -> Vec { + let mut v = Vec::new(); + f(&mut v); + v + } + + #[test] + fn read_string_round_trips() { + let buf = enc(|v| encode::write_str(v, "héllo 🦀").unwrap()); + let mut bytes = Bytes::new(&buf); + assert_eq!(read_string(&mut bytes).unwrap(), "héllo 🦀"); + assert!(bytes.remaining_slice().is_empty()); + } + + #[test] + fn read_string_on_wrong_type_errors_and_consumes_marker() { + // A lone nil marker: read_string must fail *and* consume the marker so a + // following read_optional can observe end-of-input correctly. + let mut bytes = Bytes::new(&[0xc0]); + assert!(read_string(&mut bytes).is_err()); + assert!( + bytes.remaining_slice().is_empty(), + "marker byte must be consumed" + ); + } + + #[test] + fn read_converts_rmp_errors() { + let buf = enc(|v| encode::write_u64(v, 42).unwrap()); + let mut bytes = Bytes::new(&buf); + assert_eq!(read_u64(&mut bytes).unwrap(), 42); + } + + #[rstest] + #[case::bool_true(enc(|v| encode::write_bool(v, true).unwrap()), true)] + #[case::bool_false(enc(|v| encode::write_bool(v, false).unwrap()), false)] + fn read_bool_round_trips(#[case] buf: Vec, #[case] expected: bool) { + let mut b = Bytes::new(&buf); + assert_eq!(read_bool(&mut b).unwrap(), expected); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn read_u8_round_trips() { + let buf = enc(|v| encode::write_u8(v, 200).unwrap()); + let mut b = Bytes::new(&buf); + assert_eq!(read_u8(&mut b).unwrap(), 200); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn read_u16_round_trips() { + let buf = enc(|v| encode::write_u16(v, 40000).unwrap()); + let mut b = Bytes::new(&buf); + assert_eq!(read_u16(&mut b).unwrap(), 40000); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn read_u64_round_trips() { + let buf = enc(|v| encode::write_u64(v, 42).unwrap()); + let mut b = Bytes::new(&buf); + assert_eq!(read_u64(&mut b).unwrap(), 42); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn read_i64_round_trips() { + let buf = enc(|v| { + encode::write_sint(v, -123456789).unwrap(); + }); + let mut b = Bytes::new(&buf); + assert_eq!(read_i64(&mut b).unwrap(), -123456789); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn read_optional_some_and_none() { + let some = enc(|v| encode::write_u64(v, 7).unwrap()); + let mut b = Bytes::new(&some); + assert_eq!(read_optional(&mut b, read_u64).unwrap(), Some(7)); + + let none = enc(|v| rmp::encode::write_nil(v).unwrap()); + let mut b = Bytes::new(&none); + assert_eq!(read_optional(&mut b, read_u64).unwrap(), None); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn read_optional_string() { + let buf = enc(|v| encode::write_str(v, "x").unwrap()); + let mut b = Bytes::new(&buf); + assert_eq!( + read_optional(&mut b, read_string).unwrap(), + Some("x".to_string()) + ); + } + + #[test] + fn read_optional_forwards_non_nil_errors() { + // A bool where a u64 is expected is a type mismatch that is NOT nil. + let buf = enc(|v| encode::write_bool(v, true).unwrap()); + let mut b = Bytes::new(&buf); + assert!(read_optional(&mut b, read_u64).is_err()); + } + + fn array_of(len: u32) -> Vec { + enc(|v| { + encode::write_array_len(v, len).unwrap(); + }) + } + + #[test] + fn expect_array_len_exact_ok() { + let buf = array_of(3); + let mut b = Bytes::new(&buf); + assert_eq!(expect_array_len(&mut b, 3).unwrap(), 3); + } + + #[test] + fn expect_array_len_mismatch_reports_expected_and_actual() { + let buf = array_of(5); + let mut b = Bytes::new(&buf); + match expect_array_len(&mut b, 3) { + Err(DecodeError::UnexpectedArrayLen { expected, actual }) => { + assert_eq!((expected, actual), (3, 5)); + } + other => panic!("expected UnexpectedArrayLen, got {other:?}"), + } + } + + #[test] + fn read_array_len_returns_count_for_manual_range_checks() { + let buf = array_of(9); + let mut b = Bytes::new(&buf); + assert_eq!(read_array_len(&mut b).unwrap(), 9); + } + + #[test] + fn expect_eof_ok_when_consumed() { + let buf = enc(|v| encode::write_u8(v, 1).unwrap()); + let mut b = Bytes::new(&buf); + read_u8(&mut b).unwrap(); + assert!(expect_eof(&b).is_ok()); + } + + #[test] + fn expect_eof_reports_remaining() { + let b = Bytes::new(&[0x01, 0x02, 0x03]); + match expect_eof(&b) { + Err(DecodeError::TrailingBytes { remaining }) => assert_eq!(remaining, 3), + other => panic!("expected TrailingBytes, got {other:?}"), + } + } + + #[test] + fn read_optional_nil_string_advances_to_next_field() { + // A nil optional-string field followed by a u64. The nil read must consume + // exactly the marker so the following field decodes correctly. + let mut out = Vec::new(); + encode::write_optional::<_, &str>(&mut out, None, encode::write_str).unwrap(); + encode::write_u64(&mut out, 1234).unwrap(); + + let mut b = Bytes::new(&out); + assert_eq!(read_optional(&mut b, read_string).unwrap(), None); + assert_eq!(read_u64(&mut b).unwrap(), 1234); + assert!(expect_eof(&b).is_ok()); + } + + use proptest::prelude::*; + + proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + // A string survives an encode -> read_string round trip exactly. + #[test] + fn read_string_proptest_round_trips(s in r"(?s).*") { + let buf = enc(|v| encode::write_str(v, &s).unwrap()); + let mut b = Bytes::new(&buf); + prop_assert_eq!(read_string(&mut b).unwrap(), s); + prop_assert!(b.remaining_slice().is_empty()); + } + + // Option survives write_optional -> read_optional. + #[test] + fn read_optional_u64_round_trips(v in proptest::option::of(any::())) { + let mut out = Vec::new(); + encode::write_optional(&mut out, v, encode::write_u64).unwrap(); + let mut b = Bytes::new(&out); + prop_assert_eq!(read_optional(&mut b, read_u64).unwrap(), v); + } + + // Option survives the optional round trip. + #[test] + fn read_optional_string_round_trips(v in proptest::option::of(r"(?s).*")) { + let mut out = Vec::new(); + encode::write_optional(&mut out, v.as_deref(), encode::write_str).unwrap(); + let mut b = Bytes::new(&out); + prop_assert_eq!(read_optional(&mut b, read_string).unwrap(), v); + prop_assert!(b.remaining_slice().is_empty()); + } + + // A full array record (len + fields + optional tail) round trips, and the + // cursor is exactly exhausted afterwards. + #[test] + fn record_round_trips( + id in r"[a-z]{0,16}", + ts in any::(), + deleted in proptest::option::of(any::()), + ) { + let mut out = Vec::new(); + encode::write_array_len(&mut out, 3).unwrap(); + encode::write_str(&mut out, &id).unwrap(); + encode::write_u64(&mut out, ts).unwrap(); + encode::write_optional(&mut out, deleted, encode::write_u64).unwrap(); + + let mut b = Bytes::new(&out); + prop_assert_eq!(expect_array_len(&mut b, 3).unwrap(), 3); + prop_assert_eq!(read_string(&mut b).unwrap(), id); + prop_assert_eq!(read_u64(&mut b).unwrap(), ts); + prop_assert_eq!(read_optional(&mut b, read_u64).unwrap(), deleted); + prop_assert!(expect_eof(&b).is_ok()); + } + + // Reads never panic on arbitrary bytes — they return Err instead. + #[test] + fn reads_never_panic(raw in proptest::collection::vec(any::(), 0..64)) { + let mut b = Bytes::new(&raw); + let _ = read_string(&mut b); + let mut b = Bytes::new(&raw); + let _ = read_array_len(&mut b); + let mut b = Bytes::new(&raw); + let _ = read_optional(&mut b, read_u64); + } + } +} diff --git a/crates/atuin-common/src/rmp/encode.rs b/crates/atuin-common/src/rmp/encode.rs new file mode 100644 index 00000000000..39afeff7e13 --- /dev/null +++ b/crates/atuin-common/src/rmp/encode.rs @@ -0,0 +1,79 @@ +//! MessagePack encode helpers built on [`rmp::encode`]. +//! +//! Re-exports `rmp::encode`'s `write_*` primitives so encoders depend only on +//! this module, wraps their error in [`EncodeError`] (which converts cleanly +//! into [`eyre::Report`]), and adds a nil-aware [`write_optional`]. + +use rmp::encode as mp; +use rmp::encode::{RmpWrite, RmpWriteErr, ValueWriteError}; + +// Re-export rmp's write primitives so decoders/encoders depend only on this module. +pub use rmp::encode::{ + write_array_len, write_bin, write_bool, write_sint, write_str, write_u8, write_u16, write_u64, + write_uint, +}; + +/// An error encountered while encoding a MessagePack value. +/// +/// Wraps [`ValueWriteError`] with a message that names the failing write and its +/// inner I/O error — neither of which `rmp`'s own [`Display`] reports. Implements +/// [`std::error::Error`], so it converts into [`eyre::Report`] with `?`. +/// +/// [`Display`]: std::fmt::Display +#[derive(Debug, derive_more::Display, derive_more::From, thiserror::Error)] +#[display("could not write MessagePack value: {_0:?}")] +pub struct EncodeError(ValueWriteError); + +/// Write an optional value, encoding [`None`] as a nil marker. `write` encodes +/// the inner value when present (e.g. [`write_u64`] or [`write_str`]). +pub fn write_optional( + out: &mut W, + value: Option, + write: impl FnOnce(&mut W, T) -> Result<(), ValueWriteError>, +) -> Result<(), EncodeError> { + match value { + Some(v) => write(out, v).map_err(EncodeError::from), + None => mp::write_nil(out) + .map_err(|e| EncodeError::from(ValueWriteError::InvalidMarkerWrite(e))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rmp::decode::{self, Bytes}; + use pretty_assertions::assert_eq; + + #[test] + fn write_optional_some_then_read_back() { + let mut out = Vec::new(); + write_optional(&mut out, Some(99u64), write_u64).unwrap(); + let mut b = Bytes::new(&out); + assert_eq!( + decode::read_optional(&mut b, decode::read_u64).unwrap(), + Some(99) + ); + } + + #[test] + fn write_optional_none_writes_nil() { + let mut out = Vec::new(); + write_optional::<_, u64>(&mut out, None, write_u64).unwrap(); + let mut b = Bytes::new(&out); + assert_eq!( + decode::read_optional(&mut b, decode::read_u64).unwrap(), + None + ); + } + + #[test] + fn write_optional_str() { + let mut out = Vec::new(); + write_optional(&mut out, Some("hi"), write_str).unwrap(); + let mut b = Bytes::new(&out); + assert_eq!( + decode::read_optional(&mut b, decode::read_string).unwrap(), + Some("hi".to_string()) + ); + } +} diff --git a/crates/atuin-common/src/rmp/mod.rs b/crates/atuin-common/src/rmp/mod.rs new file mode 100644 index 00000000000..cbf9c8e732a --- /dev/null +++ b/crates/atuin-common/src/rmp/mod.rs @@ -0,0 +1,10 @@ +//! atuin's MessagePack helpers: the [`decode`] and [`encode`] submodules mirror +//! `rmp`'s own module layout, but every `read_*`/`write_*` returns our own +//! [`DecodeError`]/[`EncodeError`] (which convert into [`eyre::Report`]) and the +//! modules add owned-string, optional, array-length and end-of-input helpers. + +pub mod decode; +pub mod encode; + +pub use decode::DecodeError; +pub use encode::EncodeError; diff --git a/crates/atuin-dotfiles/src/shell.rs b/crates/atuin-dotfiles/src/shell.rs index 708ccd8fc3d..c4fd9cf7dc6 100644 --- a/crates/atuin-dotfiles/src/shell.rs +++ b/crates/atuin-dotfiles/src/shell.rs @@ -1,8 +1,8 @@ +use atuin_common::rmp::encode; use eyre::Result; -use rmp::encode; use serde::Serialize; -use atuin_common::rmp::{Bytes, decode, expect_array_len, expect_eof}; +use atuin_common::rmp::decode::{self, Bytes}; use atuin_common::shell::{Shell, ShellError}; use crate::store::AliasStore; @@ -43,13 +43,13 @@ impl Var { } pub fn deserialize(bytes: &mut Bytes) -> Result { - expect_array_len(bytes, 3)?; + decode::expect_array_len(bytes, 3)?; - let name = decode::(bytes)?; - let value = decode::(bytes)?; - let export = decode::(bytes)?; + let name = decode::read_string(bytes)?; + let value = decode::read_string(bytes)?; + let export = decode::read_bool(bytes)?; - expect_eof(bytes)?; + decode::expect_eof(bytes)?; Ok(Var { name, diff --git a/crates/atuin-dotfiles/src/store.rs b/crates/atuin-dotfiles/src/store.rs index 31e8c8129c8..58b21df47d7 100644 --- a/crates/atuin-dotfiles/src/store.rs +++ b/crates/atuin-dotfiles/src/store.rs @@ -6,7 +6,7 @@ use atuin_client::record::sqlite_store::SqliteStore; // While we will support a range of shell config, I'd rather have a larger number of small records // + stores, rather than one mega config store. use atuin_common::record::{DecryptedData, Host, HostId}; -use atuin_common::rmp::{Bytes, decode, expect_array_len, expect_eof}; +use atuin_common::rmp::decode::{self, Bytes}; use atuin_common::utils::unquote; use eyre::{Result, bail, eyre}; @@ -30,7 +30,7 @@ pub enum AliasRecord { impl AliasRecord { pub fn serialize(&self) -> Result { - use rmp::encode; + use atuin_common::rmp::encode; let mut output = vec![]; @@ -58,28 +58,28 @@ impl AliasRecord { CONFIG_SHELL_ALIAS_VERSION => { let mut bytes = Bytes::new(&data.0); - let record_type = decode::(&mut bytes)?; + let record_type = decode::read_u8(&mut bytes)?; match record_type { // create 0 => { - expect_array_len(&mut bytes, 2)?; + decode::expect_array_len(&mut bytes, 2)?; - let name = decode::(&mut bytes)?; - let value = decode::(&mut bytes)?; + let name = decode::read_string(&mut bytes)?; + let value = decode::read_string(&mut bytes)?; - expect_eof(&bytes)?; + decode::expect_eof(&bytes)?; Ok(AliasRecord::Create(Alias { name, value })) } // delete 1 => { - expect_array_len(&mut bytes, 1)?; + decode::expect_array_len(&mut bytes, 1)?; - let key = decode::(&mut bytes)?; + let key = decode::read_string(&mut bytes)?; - expect_eof(&bytes)?; + decode::expect_eof(&bytes)?; Ok(AliasRecord::Delete(key)) } diff --git a/crates/atuin-dotfiles/src/store/var.rs b/crates/atuin-dotfiles/src/store/var.rs index 0aec7c3e833..53cbdf88c69 100644 --- a/crates/atuin-dotfiles/src/store/var.rs +++ b/crates/atuin-dotfiles/src/store/var.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; use atuin_client::record::sqlite_store::SqliteStore; use atuin_common::record::{DecryptedData, Host, HostId}; -use atuin_common::rmp::{Bytes, decode, expect_array_len, expect_eof}; +use atuin_common::rmp::decode::{self, Bytes}; use eyre::{Result, bail, eyre}; use atuin_client::record::encryption::PASETO_V4; @@ -26,7 +26,7 @@ pub enum VarRecord { impl VarRecord { pub fn serialize(&self) -> Result { - use rmp::encode; + use atuin_common::rmp::encode; let mut output = vec![]; @@ -52,7 +52,7 @@ impl VarRecord { DOTFILES_VAR_VERSION => { let mut bytes = Bytes::new(&data.0); - let record_type = decode::(&mut bytes)?; + let record_type = decode::read_u8(&mut bytes)?; match record_type { // create @@ -63,11 +63,11 @@ impl VarRecord { // delete 1 => { - expect_array_len(&mut bytes, 1)?; + decode::expect_array_len(&mut bytes, 1)?; - let key = decode::(&mut bytes)?; + let key = decode::read_string(&mut bytes)?; - expect_eof(&bytes)?; + decode::expect_eof(&bytes)?; Ok(VarRecord::Delete(key)) } diff --git a/crates/atuin-kv/src/store/record.rs b/crates/atuin-kv/src/store/record.rs index a39f2e8870a..ebd0096bef4 100644 --- a/crates/atuin-kv/src/store/record.rs +++ b/crates/atuin-kv/src/store/record.rs @@ -1,5 +1,5 @@ use atuin_common::record::DecryptedData; -use atuin_common::rmp::{Bytes, decode, expect_array_len, expect_eof}; +use atuin_common::rmp::decode::{self, Bytes}; use eyre::{Result, bail}; use typed_builder::TypedBuilder; @@ -16,7 +16,7 @@ pub struct KvRecord { impl KvRecord { pub fn serialize(&self) -> Result { - use rmp::encode; + use atuin_common::rmp::encode; let mut output = vec![]; @@ -39,13 +39,13 @@ impl KvRecord { "v0" => { let mut bytes = Bytes::new(&data.0); - expect_array_len(&mut bytes, 3)?; + decode::expect_array_len(&mut bytes, 3)?; - let namespace = decode::(&mut bytes)?; - let key = decode::(&mut bytes)?; - let value = decode::(&mut bytes)?; + let namespace = decode::read_string(&mut bytes)?; + let key = decode::read_string(&mut bytes)?; + let value = decode::read_string(&mut bytes)?; - expect_eof(&bytes)?; + decode::expect_eof(&bytes)?; Ok(KvRecord { namespace, @@ -56,19 +56,19 @@ impl KvRecord { KV_VERSION => { let mut bytes = Bytes::new(&data.0); - expect_array_len(&mut bytes, 4)?; + decode::expect_array_len(&mut bytes, 4)?; - let namespace = decode::(&mut bytes)?; - let key = decode::(&mut bytes)?; - let has_value = decode::(&mut bytes)?; + let namespace = decode::read_string(&mut bytes)?; + let key = decode::read_string(&mut bytes)?; + let has_value = decode::read_bool(&mut bytes)?; let value = if has_value { - Some(decode::(&mut bytes)?) + Some(decode::read_string(&mut bytes)?) } else { None }; - expect_eof(&bytes)?; + decode::expect_eof(&bytes)?; Ok(KvRecord { namespace, diff --git a/crates/atuin-scripts/src/store/record.rs b/crates/atuin-scripts/src/store/record.rs index 760a0c7dd9e..f94df270c14 100644 --- a/crates/atuin-scripts/src/store/record.rs +++ b/crates/atuin-scripts/src/store/record.rs @@ -15,7 +15,7 @@ pub enum ScriptRecord { impl ScriptRecord { pub fn serialize(&self) -> Result { - use rmp::encode; + use atuin_common::rmp::encode; let mut output = vec![]; @@ -47,33 +47,33 @@ impl ScriptRecord { } pub fn deserialize(data: &DecryptedData, version: &str) -> Result { - use atuin_common::rmp::{Bytes, decode, decode_bin_len}; + use atuin_common::rmp::decode::{self, Bytes}; match version { SCRIPT_VERSION => { let mut bytes = Bytes::new(&data.0); - let record_type = decode::(&mut bytes)?; + let record_type = decode::read_u8(&mut bytes)?; match record_type { // create 0 => { // written by encode::write_bin above - let _ = decode_bin_len(&mut bytes)?; + let _ = decode::read_bin_len(&mut bytes)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Create(script)) } // delete 1 => { - let id = decode::(&mut bytes)?; + let id = decode::read_string(&mut bytes)?; Ok(ScriptRecord::Delete(Uuid::parse_str(&id)?)) } // update 2 => { // written by encode::write_bin above - let _ = decode_bin_len(&mut bytes)?; + let _ = decode::read_bin_len(&mut bytes)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Update(script)) } diff --git a/crates/atuin-scripts/src/store/script.rs b/crates/atuin-scripts/src/store/script.rs index ce5b078e06f..7cfb9f5047f 100644 --- a/crates/atuin-scripts/src/store/script.rs +++ b/crates/atuin-scripts/src/store/script.rs @@ -2,7 +2,7 @@ use atuin_common::record::DecryptedData; use eyre::Result; use uuid::Uuid; -use rmp::encode; +use atuin_common::rmp::encode; use typed_builder::TypedBuilder; pub const SCRIPT_VERSION: &str = "v0"; @@ -60,28 +60,28 @@ impl Script { } pub fn deserialize(bytes: &[u8]) -> Result { - use atuin_common::rmp::{Bytes, decode, decode_array_len, expect_array_len, expect_eof}; + use atuin_common::rmp::decode::{self, Bytes}; let mut bytes = Bytes::new(bytes); - expect_array_len(&mut bytes, 6)?; + decode::expect_array_len(&mut bytes, 6)?; - let id = decode::(&mut bytes)?; - let name = decode::(&mut bytes)?; - let description = decode::(&mut bytes)?; - let shebang = decode::(&mut bytes)?; + let id = decode::read_string(&mut bytes)?; + let name = decode::read_string(&mut bytes)?; + let description = decode::read_string(&mut bytes)?; + let shebang = decode::read_string(&mut bytes)?; - let tags_len = decode_array_len(&mut bytes)?; + let tags_len = decode::read_array_len(&mut bytes)?; let mut tags = Vec::new(); for _ in 0..tags_len { - let tag = decode::(&mut bytes)?; + let tag = decode::read_string(&mut bytes)?; tags.push(tag); } - let script = decode::(&mut bytes)?; + let script = decode::read_string(&mut bytes)?; - expect_eof(&bytes)?; + decode::expect_eof(&bytes)?; Ok(Script { id: Uuid::parse_str(&id)?, From 7c8f79539978a5bd18b7cd8c8c815795ff75828f Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 19 Jul 2026 00:13:07 -0700 Subject: [PATCH 20/27] refactor: call rmp facade via rmp::decode/rmp::encode module paths --- Cargo.lock | 4 -- crates/atuin-client/Cargo.toml | 1 - crates/atuin-client/src/encryption.rs | 24 ++++----- crates/atuin-client/src/history.rs | 69 ++++++++++++------------ crates/atuin-client/src/history/store.rs | 21 ++++---- crates/atuin-common/src/rmp/decode.rs | 4 +- crates/atuin-dotfiles/Cargo.toml | 1 - crates/atuin-dotfiles/src/shell.rs | 23 ++++---- crates/atuin-dotfiles/src/store.rs | 36 ++++++------- crates/atuin-dotfiles/src/store/var.rs | 22 ++++---- crates/atuin-kv/Cargo.toml | 1 - crates/atuin-kv/src/store/record.rs | 40 +++++++------- crates/atuin-scripts/Cargo.toml | 1 - crates/atuin-scripts/src/store/record.rs | 31 +++++------ crates/atuin-scripts/src/store/script.rs | 40 +++++++------- 15 files changed, 148 insertions(+), 170 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 702daf1a015..dac83fa8f8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -362,7 +362,6 @@ dependencies = [ "rand 0.8.7", "regex", "reqwest", - "rmp", "rstest", "rusty_paserk", "rusty_paseto", @@ -463,7 +462,6 @@ dependencies = [ "derive_more", "eyre", "rand 0.8.7", - "rmp", "rstest", "serde", "tokio", @@ -493,7 +491,6 @@ dependencies = [ "derive_more", "eyre", "pretty_assertions", - "rmp", "sqlx", "tokio", "tracing", @@ -551,7 +548,6 @@ dependencies = [ "eyre", "minijinja", "pretty_assertions", - "rmp", "serde", "serde_json", "sql-builder", diff --git a/crates/atuin-client/Cargo.toml b/crates/atuin-client/Cargo.toml index f246e073657..4562d88fb0c 100644 --- a/crates/atuin-client/Cargo.toml +++ b/crates/atuin-client/Cargo.toml @@ -47,7 +47,6 @@ serde_regex = "1.1.0" fs-err = { workspace = true } sql-builder = { workspace = true } memchr = "2.7" -rmp = { version = "0.8.14" } typed-builder = { workspace = true } tokio = { workspace = true } semver = { workspace = true } diff --git a/crates/atuin-client/src/encryption.rs b/crates/atuin-client/src/encryption.rs index c9ead0ca8ed..8b73801382a 100644 --- a/crates/atuin-client/src/encryption.rs +++ b/crates/atuin-client/src/encryption.rs @@ -15,12 +15,9 @@ pub use crypto_secretbox::Key; use crypto_secretbox::{KeyInit, XSalsa20Poly1305, aead::OsRng}; use eyre::{Context, Result, bail, ensure}; use fs_err as fs; -use rmp::Marker; - -use atuin_common::rmp::decode::{self, Bytes}; -use atuin_common::rmp::encode; use crate::settings::Settings; +use atuin_common::rmp; pub fn generate_encoded_key() -> Result<(Key, String)> { let key = XSalsa20Poly1305::generate_key(&mut OsRng); @@ -60,10 +57,11 @@ pub fn load_key(settings: &Settings) -> Result { pub fn encode_key(key: &Key) -> Result { let mut buf = vec![]; - encode::write_array_len(&mut buf, key.len() as u32) + rmp::encode::write_array_len(&mut buf, key.len() as u32) .wrap_err("could not encode key to message pack")?; for b in key { - encode::write_uint(&mut buf, *b as u64).wrap_err("could not encode key to message pack")?; + rmp::encode::write_uint(&mut buf, *b as u64) + .wrap_err("could not encode key to message pack")?; } let buf = BASE64_STANDARD.encode(buf); @@ -80,22 +78,22 @@ pub fn decode_key(key: String) -> Result { match <[u8; 32]>::try_from(&*buf) { Ok(key) => Ok(key.into()), Err(_) => { - let mut bytes = Bytes::new(&buf); + let mut bytes = rmp::decode::Bytes::new(&buf); - match Marker::from_u8(buf[0]) { - Marker::Bin8 => { - let len = decode::read_bin_len(&mut bytes)?; + match rmp::decode::Marker::from_u8(buf[0]) { + rmp::decode::Marker::Bin8 => { + let len = rmp::decode::read_bin_len(&mut bytes)?; ensure!(len == 32, "encryption key is not the correct size"); let key = <[u8; 32]>::try_from(bytes.remaining_slice()) .context("could not decode encryption key")?; Ok(key.into()) } - Marker::Array16 => { - decode::expect_array_len(&mut bytes, 32)?; + rmp::decode::Marker::Array16 => { + rmp::decode::expect_array_len(&mut bytes, 32)?; let mut key = Key::default(); for i in &mut key { - *i = decode::read_u8(&mut bytes)?; + *i = rmp::decode::read_u8(&mut bytes)?; } Ok(key) } diff --git a/crates/atuin-client/src/history.rs b/crates/atuin-client/src/history.rs index f2d7b124775..87415c2f9d3 100644 --- a/crates/atuin-client/src/history.rs +++ b/crates/atuin-client/src/history.rs @@ -8,8 +8,7 @@ use eyre::{Result, bail}; use crate::secrets::SECRET_PATTERNS_RE; use crate::settings::Settings; use crate::utils::get_host_user; -use atuin_common::rmp::decode::{self, Bytes}; -use atuin_common::rmp::encode::{self, EncodeError}; +use atuin_common::rmp; use time::OffsetDateTime; pub(crate) mod builder; @@ -213,30 +212,30 @@ impl History { /// V2 is designed to allow new fields to be added without incrementing the version. V1 cannot /// accommodate this because its deserialization routine errors if more than 11 fields are /// provided. - pub fn serialize(&self) -> Result { + pub fn serialize(&self) -> Result { let mut output = vec![]; // write the version - encode::write_u16(&mut output, Version::LATEST.as_int())?; - encode::write_array_len(&mut output, Version::LATEST.min_fields())?; - - encode::write_str(&mut output, &self.id.0)?; - encode::write_u64(&mut output, self.timestamp.unix_timestamp_nanos() as u64)?; - encode::write_sint(&mut output, self.duration)?; - encode::write_sint(&mut output, self.exit)?; - encode::write_str(&mut output, &self.command)?; - encode::write_str(&mut output, &self.cwd)?; - encode::write_str(&mut output, &self.session)?; - encode::write_str(&mut output, &self.hostname)?; - - encode::write_optional( + rmp::encode::write_u16(&mut output, Version::LATEST.as_int())?; + rmp::encode::write_array_len(&mut output, Version::LATEST.min_fields())?; + + rmp::encode::write_str(&mut output, &self.id.0)?; + rmp::encode::write_u64(&mut output, self.timestamp.unix_timestamp_nanos() as u64)?; + rmp::encode::write_sint(&mut output, self.duration)?; + rmp::encode::write_sint(&mut output, self.exit)?; + rmp::encode::write_str(&mut output, &self.command)?; + rmp::encode::write_str(&mut output, &self.cwd)?; + rmp::encode::write_str(&mut output, &self.session)?; + rmp::encode::write_str(&mut output, &self.hostname)?; + + rmp::encode::write_optional( &mut output, self.deleted_at.map(|d| d.unix_timestamp_nanos() as u64), - encode::write_u64, + rmp::encode::write_u64, )?; - encode::write_str(&mut output, self.author.as_str())?; - encode::write_optional(&mut output, self.intent.as_deref(), encode::write_str)?; - encode::write_optional(&mut output, self.shell.as_deref(), encode::write_str)?; + rmp::encode::write_str(&mut output, self.author.as_str())?; + rmp::encode::write_optional(&mut output, self.intent.as_deref(), rmp::encode::write_str)?; + rmp::encode::write_optional(&mut output, self.shell.as_deref(), rmp::encode::write_str)?; Ok(DecryptedData(output)) } @@ -245,32 +244,32 @@ impl History { bail!("unknown version {version:?}"); }; - let mut bytes = Bytes::new(bytes); + let mut bytes = rmp::decode::Bytes::new(bytes); - let real_version = decode::read_u16(&mut bytes)?; + let real_version = rmp::decode::read_u16(&mut bytes)?; if real_version != version.as_int() { bail!("expected to decode {version} record, found v{real_version}"); } - let nfields = decode::read_array_len(&mut bytes)?; + let nfields = rmp::decode::read_array_len(&mut bytes)?; let min_fields = version.min_fields(); if nfields < min_fields || version.max_fields().is_some_and(|max| nfields > max) { bail!("unexpected number of fields ({nfields}) for history version {version}"); } - let id = decode::read_string(&mut bytes)?; - let timestamp = decode::read_u64(&mut bytes)?; - let duration = decode::read_i64(&mut bytes)?; - let exit = decode::read_i64(&mut bytes)?; + let id = rmp::decode::read_string(&mut bytes)?; + let timestamp = rmp::decode::read_u64(&mut bytes)?; + let duration = rmp::decode::read_i64(&mut bytes)?; + let exit = rmp::decode::read_i64(&mut bytes)?; - let command = decode::read_string(&mut bytes)?; - let cwd = decode::read_string(&mut bytes)?; - let session = decode::read_string(&mut bytes)?; - let hostname = decode::read_string(&mut bytes)?; - let deleted_at = decode::read_optional(&mut bytes, decode::read_u64)?; + let command = rmp::decode::read_string(&mut bytes)?; + let cwd = rmp::decode::read_string(&mut bytes)?; + let session = rmp::decode::read_string(&mut bytes)?; + let hostname = rmp::decode::read_string(&mut bytes)?; + let deleted_at = rmp::decode::read_optional(&mut bytes, rmp::decode::read_u64)?; let author = if version >= Version::One { - decode::read_optional(&mut bytes, decode::read_string)? + rmp::decode::read_optional(&mut bytes, rmp::decode::read_string)? } else { None }; @@ -280,13 +279,13 @@ impl History { Version::One => nfields > min_fields, Version::Two => true, } { - decode::read_optional(&mut bytes, decode::read_string)? + rmp::decode::read_optional(&mut bytes, rmp::decode::read_string)? } else { None }; let shell = if version >= Version::Two { - decode::read_optional(&mut bytes, decode::read_string)? + rmp::decode::read_optional(&mut bytes, rmp::decode::read_string)? } else { None }; diff --git a/crates/atuin-client/src/history/store.rs b/crates/atuin-client/src/history/store.rs index 7deab24583d..09ca993a696 100644 --- a/crates/atuin-client/src/history/store.rs +++ b/crates/atuin-client/src/history/store.rs @@ -9,9 +9,9 @@ use crate::{ record::{encryption::PASETO_V4, sqlite_store::SqliteStore, store::Store}, }; use atuin_common::record::{DecryptedData, Host, HostId, Record, RecordId, RecordIdx}; -use atuin_common::rmp::decode::{self, Bytes}; use super::{HISTORY_TAG, History, HistoryId, Version}; +use atuin_common::rmp; #[derive(Debug, Clone)] pub struct HistoryStore { @@ -47,23 +47,22 @@ impl HistoryRecord { pub fn serialize(&self) -> Result { // probably don't actually need to use rmp here, but if we ever need to extend it, it's a // nice wrapper around raw byte stuff - use atuin_common::rmp::encode; let mut output = vec![]; match self { HistoryRecord::Create(history) => { // 0 -> a history create - encode::write_u8(&mut output, 0)?; + rmp::encode::write_u8(&mut output, 0)?; let bytes = history.serialize()?; - encode::write_bin(&mut output, &bytes.0)?; + rmp::encode::write_bin(&mut output, &bytes.0)?; } HistoryRecord::Delete(id) => { // 1 -> a history delete - encode::write_u8(&mut output, 1)?; - encode::write_str(&mut output, id.0.as_str())?; + rmp::encode::write_u8(&mut output, 1)?; + rmp::encode::write_str(&mut output, id.0.as_str())?; } }; @@ -71,16 +70,16 @@ impl HistoryRecord { } pub fn deserialize(bytes: &DecryptedData, version: &str) -> Result { - let mut bytes = Bytes::new(&bytes.0); + let mut bytes = rmp::decode::Bytes::new(&bytes.0); - let record_type = decode::read_u8(&mut bytes)?; + let record_type = rmp::decode::read_u8(&mut bytes)?; match record_type { // 0 -> HistoryRecord::Create 0 => { // not super useful to us atm, but perhaps in the future // written by write_bin above - let _ = decode::read_bin_len(&mut bytes)?; + let _ = rmp::decode::read_bin_len(&mut bytes)?; let record = History::deserialize(bytes.remaining_slice(), version)?; @@ -89,8 +88,8 @@ impl HistoryRecord { // 1 -> HistoryRecord::Delete 1 => { - let id = decode::read_string(&mut bytes)?; - decode::expect_eof(&bytes)?; + let id = rmp::decode::read_string(&mut bytes)?; + rmp::decode::expect_eof(&bytes)?; Ok(HistoryRecord::Delete(id.into())) } diff --git a/crates/atuin-common/src/rmp/decode.rs b/crates/atuin-common/src/rmp/decode.rs index ab39ee91f66..addde493a82 100644 --- a/crates/atuin-common/src/rmp/decode.rs +++ b/crates/atuin-common/src/rmp/decode.rs @@ -7,7 +7,6 @@ //! [`read_array_len`]/[`expect_array_len`] and [`expect_eof`] that `rmp` does //! not perform itself. -use rmp::Marker; use rmp::decode as mp; use rmp::decode::bytes::BytesReadError; use rmp::decode::{DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, ValueReadError}; @@ -15,6 +14,9 @@ use rmp::decode::{DecodeStringError, NumValueReadError, RmpRead, RmpReadErr, Val /// Re-exported so decoders need only depend on this module, not `rmp::decode`. pub use rmp::decode::bytes::Bytes; +/// Re-exported so decoders can match on markers via this module, not `rmp`. +pub use rmp::Marker; + /// An error encountered while decoding a MessagePack value. /// /// Wraps the three error types `rmp`'s decode functions return, and adds two diff --git a/crates/atuin-dotfiles/Cargo.toml b/crates/atuin-dotfiles/Cargo.toml index 53f7a213f6b..6b51445fd15 100644 --- a/crates/atuin-dotfiles/Cargo.toml +++ b/crates/atuin-dotfiles/Cargo.toml @@ -21,7 +21,6 @@ derive_more = { workspace = true } eyre = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } -rmp = { version = "0.8.14" } rand = { workspace = true } serde = { workspace = true } crypto_secretbox = "0.1.1" diff --git a/crates/atuin-dotfiles/src/shell.rs b/crates/atuin-dotfiles/src/shell.rs index c4fd9cf7dc6..c5bcf38c4c3 100644 --- a/crates/atuin-dotfiles/src/shell.rs +++ b/crates/atuin-dotfiles/src/shell.rs @@ -1,8 +1,7 @@ -use atuin_common::rmp::encode; use eyre::Result; use serde::Serialize; -use atuin_common::rmp::decode::{self, Bytes}; +use atuin_common::rmp; use atuin_common::shell::{Shell, ShellError}; use crate::store::AliasStore; @@ -33,23 +32,23 @@ impl Var { /// Serialize into the given vec /// This is intended to be called by the store pub fn serialize(&self, output: &mut Vec) -> Result<()> { - encode::write_array_len(output, 3)?; // 3 fields + rmp::encode::write_array_len(output, 3)?; // 3 fields - encode::write_str(output, self.name.as_str())?; - encode::write_str(output, self.value.as_str())?; - encode::write_bool(output, self.export)?; + rmp::encode::write_str(output, self.name.as_str())?; + rmp::encode::write_str(output, self.value.as_str())?; + rmp::encode::write_bool(output, self.export)?; Ok(()) } - pub fn deserialize(bytes: &mut Bytes) -> Result { - decode::expect_array_len(bytes, 3)?; + pub fn deserialize(bytes: &mut rmp::decode::Bytes) -> Result { + rmp::decode::expect_array_len(bytes, 3)?; - let name = decode::read_string(bytes)?; - let value = decode::read_string(bytes)?; - let export = decode::read_bool(bytes)?; + let name = rmp::decode::read_string(bytes)?; + let value = rmp::decode::read_string(bytes)?; + let export = rmp::decode::read_bool(bytes)?; - decode::expect_eof(bytes)?; + rmp::decode::expect_eof(bytes)?; Ok(Var { name, diff --git a/crates/atuin-dotfiles/src/store.rs b/crates/atuin-dotfiles/src/store.rs index 58b21df47d7..ecb69e38dbc 100644 --- a/crates/atuin-dotfiles/src/store.rs +++ b/crates/atuin-dotfiles/src/store.rs @@ -6,7 +6,7 @@ use atuin_client::record::sqlite_store::SqliteStore; // While we will support a range of shell config, I'd rather have a larger number of small records // + stores, rather than one mega config store. use atuin_common::record::{DecryptedData, Host, HostId}; -use atuin_common::rmp::decode::{self, Bytes}; +use atuin_common::rmp; use atuin_common::utils::unquote; use eyre::{Result, bail, eyre}; @@ -30,23 +30,21 @@ pub enum AliasRecord { impl AliasRecord { pub fn serialize(&self) -> Result { - use atuin_common::rmp::encode; - let mut output = vec![]; match self { AliasRecord::Create(alias) => { - encode::write_u8(&mut output, 0)?; // create - encode::write_array_len(&mut output, 2)?; // 2 fields + rmp::encode::write_u8(&mut output, 0)?; // create + rmp::encode::write_array_len(&mut output, 2)?; // 2 fields - encode::write_str(&mut output, alias.name.as_str())?; - encode::write_str(&mut output, alias.value.as_str())?; + rmp::encode::write_str(&mut output, alias.name.as_str())?; + rmp::encode::write_str(&mut output, alias.value.as_str())?; } AliasRecord::Delete(name) => { - encode::write_u8(&mut output, 1)?; // delete - encode::write_array_len(&mut output, 1)?; // 1 field + rmp::encode::write_u8(&mut output, 1)?; // delete + rmp::encode::write_array_len(&mut output, 1)?; // 1 field - encode::write_str(&mut output, name.as_str())?; + rmp::encode::write_str(&mut output, name.as_str())?; } } @@ -56,30 +54,30 @@ impl AliasRecord { pub fn deserialize(data: &DecryptedData, version: &str) -> Result { match version { CONFIG_SHELL_ALIAS_VERSION => { - let mut bytes = Bytes::new(&data.0); + let mut bytes = rmp::decode::Bytes::new(&data.0); - let record_type = decode::read_u8(&mut bytes)?; + let record_type = rmp::decode::read_u8(&mut bytes)?; match record_type { // create 0 => { - decode::expect_array_len(&mut bytes, 2)?; + rmp::decode::expect_array_len(&mut bytes, 2)?; - let name = decode::read_string(&mut bytes)?; - let value = decode::read_string(&mut bytes)?; + let name = rmp::decode::read_string(&mut bytes)?; + let value = rmp::decode::read_string(&mut bytes)?; - decode::expect_eof(&bytes)?; + rmp::decode::expect_eof(&bytes)?; Ok(AliasRecord::Create(Alias { name, value })) } // delete 1 => { - decode::expect_array_len(&mut bytes, 1)?; + rmp::decode::expect_array_len(&mut bytes, 1)?; - let key = decode::read_string(&mut bytes)?; + let key = rmp::decode::read_string(&mut bytes)?; - decode::expect_eof(&bytes)?; + rmp::decode::expect_eof(&bytes)?; Ok(AliasRecord::Delete(key)) } diff --git a/crates/atuin-dotfiles/src/store/var.rs b/crates/atuin-dotfiles/src/store/var.rs index 53cbdf88c69..f55d6dae6a4 100644 --- a/crates/atuin-dotfiles/src/store/var.rs +++ b/crates/atuin-dotfiles/src/store/var.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; use atuin_client::record::sqlite_store::SqliteStore; use atuin_common::record::{DecryptedData, Host, HostId}; -use atuin_common::rmp::decode::{self, Bytes}; +use atuin_common::rmp; use eyre::{Result, bail, eyre}; use atuin_client::record::encryption::PASETO_V4; @@ -26,21 +26,19 @@ pub enum VarRecord { impl VarRecord { pub fn serialize(&self) -> Result { - use atuin_common::rmp::encode; - let mut output = vec![]; match self { VarRecord::Create(env) => { - encode::write_u8(&mut output, 0)?; // create + rmp::encode::write_u8(&mut output, 0)?; // create env.serialize(&mut output)?; } VarRecord::Delete(env) => { - encode::write_u8(&mut output, 1)?; // delete - encode::write_array_len(&mut output, 1)?; // 1 field + rmp::encode::write_u8(&mut output, 1)?; // delete + rmp::encode::write_array_len(&mut output, 1)?; // 1 field - encode::write_str(&mut output, env.as_str())?; + rmp::encode::write_str(&mut output, env.as_str())?; } } @@ -50,9 +48,9 @@ impl VarRecord { pub fn deserialize(data: &DecryptedData, version: &str) -> Result { match version { DOTFILES_VAR_VERSION => { - let mut bytes = Bytes::new(&data.0); + let mut bytes = rmp::decode::Bytes::new(&data.0); - let record_type = decode::read_u8(&mut bytes)?; + let record_type = rmp::decode::read_u8(&mut bytes)?; match record_type { // create @@ -63,11 +61,11 @@ impl VarRecord { // delete 1 => { - decode::expect_array_len(&mut bytes, 1)?; + rmp::decode::expect_array_len(&mut bytes, 1)?; - let key = decode::read_string(&mut bytes)?; + let key = rmp::decode::read_string(&mut bytes)?; - decode::expect_eof(&bytes)?; + rmp::decode::expect_eof(&bytes)?; Ok(VarRecord::Delete(key)) } diff --git a/crates/atuin-kv/Cargo.toml b/crates/atuin-kv/Cargo.toml index 9e74f848877..45e5e189f05 100644 --- a/crates/atuin-kv/Cargo.toml +++ b/crates/atuin-kv/Cargo.toml @@ -20,7 +20,6 @@ derive_more = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } -rmp = { version = "0.8.14" } eyre = { workspace = true } tokio = { workspace = true } typed-builder = { workspace = true } diff --git a/crates/atuin-kv/src/store/record.rs b/crates/atuin-kv/src/store/record.rs index ebd0096bef4..1090bd428dd 100644 --- a/crates/atuin-kv/src/store/record.rs +++ b/crates/atuin-kv/src/store/record.rs @@ -1,5 +1,5 @@ use atuin_common::record::DecryptedData; -use atuin_common::rmp::decode::{self, Bytes}; +use atuin_common::rmp; use eyre::{Result, bail}; use typed_builder::TypedBuilder; @@ -16,19 +16,17 @@ pub struct KvRecord { impl KvRecord { pub fn serialize(&self) -> Result { - use atuin_common::rmp::encode; - let mut output = vec![]; // INFO: ensure this is updated when adding new fields - encode::write_array_len(&mut output, 4)?; + rmp::encode::write_array_len(&mut output, 4)?; - encode::write_str(&mut output, &self.namespace)?; - encode::write_str(&mut output, &self.key)?; - encode::write_bool(&mut output, self.value.is_some())?; + rmp::encode::write_str(&mut output, &self.namespace)?; + rmp::encode::write_str(&mut output, &self.key)?; + rmp::encode::write_bool(&mut output, self.value.is_some())?; if let Some(value) = &self.value { - encode::write_str(&mut output, value)?; + rmp::encode::write_str(&mut output, value)?; } Ok(DecryptedData(output)) @@ -37,15 +35,15 @@ impl KvRecord { pub fn deserialize(data: &DecryptedData, version: &str) -> Result { match version { "v0" => { - let mut bytes = Bytes::new(&data.0); + let mut bytes = rmp::decode::Bytes::new(&data.0); - decode::expect_array_len(&mut bytes, 3)?; + rmp::decode::expect_array_len(&mut bytes, 3)?; - let namespace = decode::read_string(&mut bytes)?; - let key = decode::read_string(&mut bytes)?; - let value = decode::read_string(&mut bytes)?; + let namespace = rmp::decode::read_string(&mut bytes)?; + let key = rmp::decode::read_string(&mut bytes)?; + let value = rmp::decode::read_string(&mut bytes)?; - decode::expect_eof(&bytes)?; + rmp::decode::expect_eof(&bytes)?; Ok(KvRecord { namespace, @@ -54,21 +52,21 @@ impl KvRecord { }) } KV_VERSION => { - let mut bytes = Bytes::new(&data.0); + let mut bytes = rmp::decode::Bytes::new(&data.0); - decode::expect_array_len(&mut bytes, 4)?; + rmp::decode::expect_array_len(&mut bytes, 4)?; - let namespace = decode::read_string(&mut bytes)?; - let key = decode::read_string(&mut bytes)?; - let has_value = decode::read_bool(&mut bytes)?; + let namespace = rmp::decode::read_string(&mut bytes)?; + let key = rmp::decode::read_string(&mut bytes)?; + let has_value = rmp::decode::read_bool(&mut bytes)?; let value = if has_value { - Some(decode::read_string(&mut bytes)?) + Some(rmp::decode::read_string(&mut bytes)?) } else { None }; - decode::expect_eof(&bytes)?; + rmp::decode::expect_eof(&bytes)?; Ok(KvRecord { namespace, diff --git a/crates/atuin-scripts/Cargo.toml b/crates/atuin-scripts/Cargo.toml index fb2e8cce2ee..d3e8c240bfc 100644 --- a/crates/atuin-scripts/Cargo.toml +++ b/crates/atuin-scripts/Cargo.toml @@ -20,7 +20,6 @@ derive_more = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } -rmp = { version = "0.8.14" } uuid = { workspace = true } eyre = { workspace = true } tokio = { workspace = true } diff --git a/crates/atuin-scripts/src/store/record.rs b/crates/atuin-scripts/src/store/record.rs index f94df270c14..bfe9d37bbc7 100644 --- a/crates/atuin-scripts/src/store/record.rs +++ b/crates/atuin-scripts/src/store/record.rs @@ -3,6 +3,7 @@ use eyre::{Result, eyre}; use uuid::Uuid; use crate::store::script::SCRIPT_VERSION; +use atuin_common::rmp; use super::script::Script; @@ -15,31 +16,29 @@ pub enum ScriptRecord { impl ScriptRecord { pub fn serialize(&self) -> Result { - use atuin_common::rmp::encode; - let mut output = vec![]; match self { ScriptRecord::Create(script) => { // 0 -> a script create - encode::write_u8(&mut output, 0)?; + rmp::encode::write_u8(&mut output, 0)?; let bytes = script.serialize()?; - encode::write_bin(&mut output, &bytes.0)?; + rmp::encode::write_bin(&mut output, &bytes.0)?; } ScriptRecord::Delete(id) => { // 1 -> a script delete - encode::write_u8(&mut output, 1)?; - encode::write_str(&mut output, id.to_string().as_str())?; + rmp::encode::write_u8(&mut output, 1)?; + rmp::encode::write_str(&mut output, id.to_string().as_str())?; } ScriptRecord::Update(script) => { // 2 -> a script update - encode::write_u8(&mut output, 2)?; + rmp::encode::write_u8(&mut output, 2)?; let bytes = script.serialize()?; - encode::write_bin(&mut output, &bytes.0)?; + rmp::encode::write_bin(&mut output, &bytes.0)?; } }; @@ -47,33 +46,31 @@ impl ScriptRecord { } pub fn deserialize(data: &DecryptedData, version: &str) -> Result { - use atuin_common::rmp::decode::{self, Bytes}; - match version { SCRIPT_VERSION => { - let mut bytes = Bytes::new(&data.0); + let mut bytes = rmp::decode::Bytes::new(&data.0); - let record_type = decode::read_u8(&mut bytes)?; + let record_type = rmp::decode::read_u8(&mut bytes)?; match record_type { // create 0 => { - // written by encode::write_bin above - let _ = decode::read_bin_len(&mut bytes)?; + // written by rmp::encode::write_bin above + let _ = rmp::decode::read_bin_len(&mut bytes)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Create(script)) } // delete 1 => { - let id = decode::read_string(&mut bytes)?; + let id = rmp::decode::read_string(&mut bytes)?; Ok(ScriptRecord::Delete(Uuid::parse_str(&id)?)) } // update 2 => { - // written by encode::write_bin above - let _ = decode::read_bin_len(&mut bytes)?; + // written by rmp::encode::write_bin above + let _ = rmp::decode::read_bin_len(&mut bytes)?; let script = Script::deserialize(bytes.remaining_slice())?; Ok(ScriptRecord::Update(script)) } diff --git a/crates/atuin-scripts/src/store/script.rs b/crates/atuin-scripts/src/store/script.rs index 7cfb9f5047f..19899e12897 100644 --- a/crates/atuin-scripts/src/store/script.rs +++ b/crates/atuin-scripts/src/store/script.rs @@ -2,7 +2,7 @@ use atuin_common::record::DecryptedData; use eyre::Result; use uuid::Uuid; -use atuin_common::rmp::encode; +use atuin_common::rmp; use typed_builder::TypedBuilder; pub const SCRIPT_VERSION: &str = "v0"; @@ -43,45 +43,43 @@ impl Script { let mut output = vec![]; - encode::write_array_len(&mut output, 6)?; - encode::write_str(&mut output, &self.id.to_string())?; - encode::write_str(&mut output, &self.name)?; - encode::write_str(&mut output, &self.description)?; - encode::write_str(&mut output, &self.shebang)?; - encode::write_array_len(&mut output, self.tags.len() as u32)?; + rmp::encode::write_array_len(&mut output, 6)?; + rmp::encode::write_str(&mut output, &self.id.to_string())?; + rmp::encode::write_str(&mut output, &self.name)?; + rmp::encode::write_str(&mut output, &self.description)?; + rmp::encode::write_str(&mut output, &self.shebang)?; + rmp::encode::write_array_len(&mut output, self.tags.len() as u32)?; for tag in &tags { - encode::write_str(&mut output, tag)?; + rmp::encode::write_str(&mut output, tag)?; } - encode::write_str(&mut output, &self.script)?; + rmp::encode::write_str(&mut output, &self.script)?; Ok(DecryptedData(output)) } pub fn deserialize(bytes: &[u8]) -> Result { - use atuin_common::rmp::decode::{self, Bytes}; + let mut bytes = rmp::decode::Bytes::new(bytes); - let mut bytes = Bytes::new(bytes); + rmp::decode::expect_array_len(&mut bytes, 6)?; - decode::expect_array_len(&mut bytes, 6)?; + let id = rmp::decode::read_string(&mut bytes)?; + let name = rmp::decode::read_string(&mut bytes)?; + let description = rmp::decode::read_string(&mut bytes)?; + let shebang = rmp::decode::read_string(&mut bytes)?; - let id = decode::read_string(&mut bytes)?; - let name = decode::read_string(&mut bytes)?; - let description = decode::read_string(&mut bytes)?; - let shebang = decode::read_string(&mut bytes)?; - - let tags_len = decode::read_array_len(&mut bytes)?; + let tags_len = rmp::decode::read_array_len(&mut bytes)?; let mut tags = Vec::new(); for _ in 0..tags_len { - let tag = decode::read_string(&mut bytes)?; + let tag = rmp::decode::read_string(&mut bytes)?; tags.push(tag); } - let script = decode::read_string(&mut bytes)?; + let script = rmp::decode::read_string(&mut bytes)?; - decode::expect_eof(&bytes)?; + rmp::decode::expect_eof(&bytes)?; Ok(Script { id: Uuid::parse_str(&id)?, From 5d19aa0d521867b671cb59a29d2c2dcfe2cc6aef Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 19 Jul 2026 00:30:15 -0700 Subject: [PATCH 21/27] feat(common): add read_total_array framing combinator for rmp records --- crates/atuin-common/src/rmp/decode.rs | 88 +++++++++++++++++++++++- crates/atuin-dotfiles/src/shell.rs | 18 ++--- crates/atuin-dotfiles/src/store.rs | 28 +++----- crates/atuin-dotfiles/src/store/var.rs | 12 +--- crates/atuin-kv/src/store/record.rs | 52 ++++++-------- crates/atuin-scripts/src/store/script.rs | 52 +++++++------- 6 files changed, 153 insertions(+), 97 deletions(-) diff --git a/crates/atuin-common/src/rmp/decode.rs b/crates/atuin-common/src/rmp/decode.rs index addde493a82..ebc9edcfdb7 100644 --- a/crates/atuin-common/src/rmp/decode.rs +++ b/crates/atuin-common/src/rmp/decode.rs @@ -162,9 +162,33 @@ where } } +/// Decode a MessagePack array that is expected to be the entire remaining +/// input: asserts the next value is an array of exactly `len` elements, runs +/// `read` to decode them, then asserts no trailing bytes remain. +/// +/// This bundles the array-length precondition and the end-of-input postcondition +/// around a record's field reads. `read` may return any error that a +/// [`DecodeError`] converts into (e.g. `eyre::Report`), so field decoding can mix +/// `rmp::decode::read_*` with other fallible work. +pub fn read_total_array<'a, T, E>( + bytes: &mut Bytes<'a>, + len: u32, + read: impl FnOnce(&mut Bytes<'a>) -> Result, +) -> Result +where + E: From>, +{ + expect_array_len(bytes, len).map_err(E::from)?; + let value = read(bytes)?; + expect_eof(bytes).map_err(E::from)?; + Ok(value) +} + /// Read an array-length header and require it to equal `expected`, else /// [`DecodeError::UnexpectedArrayLen`]. For a forward-compatible field count, -/// use [`read_array_len`] and range-check yourself. +/// use [`read_array_len`] and range-check yourself. For a record that is exactly +/// a whole top-level array, prefer [`read_total_array`], which also checks for +/// trailing bytes. pub fn expect_array_len<'a>(bytes: &mut Bytes<'a>, expected: u32) -> Result> { let actual = read_array_len(bytes)?; if actual == expected { @@ -175,6 +199,9 @@ pub fn expect_array_len<'a>(bytes: &mut Bytes<'a>, expected: u32) -> Result(bytes: &Bytes<'a>) -> Result<(), DecodeError<'a>> { let remaining = bytes.remaining_slice().len(); if remaining == 0 { @@ -394,6 +421,65 @@ mod tests { } } + #[test] + fn read_total_array_happy_path_reads_fields_and_consumes_all() { + let buf = enc(|v| { + encode::write_array_len(v, 2).unwrap(); + encode::write_str(v, "a").unwrap(); + encode::write_str(v, "b").unwrap(); + }); + let mut b = Bytes::new(&buf); + let (first, second): (String, String) = read_total_array(&mut b, 2, |b| { + Ok::<_, DecodeError>((read_string(b)?, read_string(b)?)) + }) + .unwrap(); + assert_eq!((first, second), ("a".to_string(), "b".to_string())); + assert!(b.remaining_slice().is_empty()); + } + + #[test] + fn read_total_array_wrong_len_reports_unexpected_array_len() { + let buf = array_of(3); + let mut b = Bytes::new(&buf); + let result: Result<(), DecodeError> = read_total_array(&mut b, 2, |_| Ok(())); + match result { + Err(DecodeError::UnexpectedArrayLen { expected, actual }) => { + assert_eq!((expected, actual), (2, 3)); + } + other => panic!("expected UnexpectedArrayLen, got {other:?}"), + } + } + + #[test] + fn read_total_array_trailing_bytes_reports_trailing() { + let mut buf = enc(|v| { + encode::write_array_len(v, 1).unwrap(); + encode::write_str(v, "x").unwrap(); + }); + buf.push(0x01); // an extra byte after the array + let mut b = Bytes::new(&buf); + let result: Result = read_total_array(&mut b, 1, |b| read_string(b)); + match result { + Err(DecodeError::TrailingBytes { remaining }) => assert_eq!(remaining, 1), + other => panic!("expected TrailingBytes, got {other:?}"), + } + } + + #[test] + fn read_total_array_supports_eyre_error_closure() { + // The closure returns eyre::Report, not DecodeError, exercising the + // `E: From` generic. read_u64's DecodeError must convert in. + let buf = enc(|v| { + encode::write_array_len(v, 1).unwrap(); + encode::write_u64(v, 99).unwrap(); + }); + let mut b = Bytes::new(&buf); + let value: u64 = + read_total_array(&mut b, 1, |b| -> eyre::Result { Ok(read_u64(b)?) }).unwrap(); + assert_eq!(value, 99); + assert!(b.remaining_slice().is_empty()); + } + #[test] fn read_optional_nil_string_advances_to_next_field() { // A nil optional-string field followed by a u64. The nil read must consume diff --git a/crates/atuin-dotfiles/src/shell.rs b/crates/atuin-dotfiles/src/shell.rs index c5bcf38c4c3..715af8ce8b3 100644 --- a/crates/atuin-dotfiles/src/shell.rs +++ b/crates/atuin-dotfiles/src/shell.rs @@ -42,18 +42,12 @@ impl Var { } pub fn deserialize(bytes: &mut rmp::decode::Bytes) -> Result { - rmp::decode::expect_array_len(bytes, 3)?; - - let name = rmp::decode::read_string(bytes)?; - let value = rmp::decode::read_string(bytes)?; - let export = rmp::decode::read_bool(bytes)?; - - rmp::decode::expect_eof(bytes)?; - - Ok(Var { - name, - value, - export, + rmp::decode::read_total_array(bytes, 3, |b| { + Ok(Var { + name: rmp::decode::read_string(b)?, + value: rmp::decode::read_string(b)?, + export: rmp::decode::read_bool(b)?, + }) }) } } diff --git a/crates/atuin-dotfiles/src/store.rs b/crates/atuin-dotfiles/src/store.rs index ecb69e38dbc..44bbd47d9dd 100644 --- a/crates/atuin-dotfiles/src/store.rs +++ b/crates/atuin-dotfiles/src/store.rs @@ -60,27 +60,17 @@ impl AliasRecord { match record_type { // create - 0 => { - rmp::decode::expect_array_len(&mut bytes, 2)?; - - let name = rmp::decode::read_string(&mut bytes)?; - let value = rmp::decode::read_string(&mut bytes)?; - - rmp::decode::expect_eof(&bytes)?; - - Ok(AliasRecord::Create(Alias { name, value })) - } + 0 => rmp::decode::read_total_array(&mut bytes, 2, |b| { + Ok(AliasRecord::Create(Alias { + name: rmp::decode::read_string(b)?, + value: rmp::decode::read_string(b)?, + })) + }), // delete - 1 => { - rmp::decode::expect_array_len(&mut bytes, 1)?; - - let key = rmp::decode::read_string(&mut bytes)?; - - rmp::decode::expect_eof(&bytes)?; - - Ok(AliasRecord::Delete(key)) - } + 1 => rmp::decode::read_total_array(&mut bytes, 1, |b| { + Ok(AliasRecord::Delete(rmp::decode::read_string(b)?)) + }), n => { bail!("unknown AliasRecord type {n}") diff --git a/crates/atuin-dotfiles/src/store/var.rs b/crates/atuin-dotfiles/src/store/var.rs index f55d6dae6a4..30959f45fa3 100644 --- a/crates/atuin-dotfiles/src/store/var.rs +++ b/crates/atuin-dotfiles/src/store/var.rs @@ -60,15 +60,9 @@ impl VarRecord { } // delete - 1 => { - rmp::decode::expect_array_len(&mut bytes, 1)?; - - let key = rmp::decode::read_string(&mut bytes)?; - - rmp::decode::expect_eof(&bytes)?; - - Ok(VarRecord::Delete(key)) - } + 1 => rmp::decode::read_total_array(&mut bytes, 1, |b| { + Ok(VarRecord::Delete(rmp::decode::read_string(b)?)) + }), n => { bail!("unknown Dotfiles var record type {n}") diff --git a/crates/atuin-kv/src/store/record.rs b/crates/atuin-kv/src/store/record.rs index 1090bd428dd..c75d43a6c20 100644 --- a/crates/atuin-kv/src/store/record.rs +++ b/crates/atuin-kv/src/store/record.rs @@ -37,41 +37,33 @@ impl KvRecord { "v0" => { let mut bytes = rmp::decode::Bytes::new(&data.0); - rmp::decode::expect_array_len(&mut bytes, 3)?; - - let namespace = rmp::decode::read_string(&mut bytes)?; - let key = rmp::decode::read_string(&mut bytes)?; - let value = rmp::decode::read_string(&mut bytes)?; - - rmp::decode::expect_eof(&bytes)?; - - Ok(KvRecord { - namespace, - key, - value: Some(value), + rmp::decode::read_total_array(&mut bytes, 3, |b| { + Ok(KvRecord { + namespace: rmp::decode::read_string(b)?, + key: rmp::decode::read_string(b)?, + value: Some(rmp::decode::read_string(b)?), + }) }) } KV_VERSION => { let mut bytes = rmp::decode::Bytes::new(&data.0); - rmp::decode::expect_array_len(&mut bytes, 4)?; - - let namespace = rmp::decode::read_string(&mut bytes)?; - let key = rmp::decode::read_string(&mut bytes)?; - let has_value = rmp::decode::read_bool(&mut bytes)?; - - let value = if has_value { - Some(rmp::decode::read_string(&mut bytes)?) - } else { - None - }; - - rmp::decode::expect_eof(&bytes)?; - - Ok(KvRecord { - namespace, - key, - value, + rmp::decode::read_total_array(&mut bytes, 4, |b| { + let namespace = rmp::decode::read_string(b)?; + let key = rmp::decode::read_string(b)?; + let has_value = rmp::decode::read_bool(b)?; + + let value = if has_value { + Some(rmp::decode::read_string(b)?) + } else { + None + }; + + Ok(KvRecord { + namespace, + key, + value, + }) }) } _ => { diff --git a/crates/atuin-scripts/src/store/script.rs b/crates/atuin-scripts/src/store/script.rs index 19899e12897..be4feb1dab1 100644 --- a/crates/atuin-scripts/src/store/script.rs +++ b/crates/atuin-scripts/src/store/script.rs @@ -62,32 +62,32 @@ impl Script { pub fn deserialize(bytes: &[u8]) -> Result { let mut bytes = rmp::decode::Bytes::new(bytes); - rmp::decode::expect_array_len(&mut bytes, 6)?; - - let id = rmp::decode::read_string(&mut bytes)?; - let name = rmp::decode::read_string(&mut bytes)?; - let description = rmp::decode::read_string(&mut bytes)?; - let shebang = rmp::decode::read_string(&mut bytes)?; - - let tags_len = rmp::decode::read_array_len(&mut bytes)?; - - let mut tags = Vec::new(); - for _ in 0..tags_len { - let tag = rmp::decode::read_string(&mut bytes)?; - tags.push(tag); - } - - let script = rmp::decode::read_string(&mut bytes)?; - - rmp::decode::expect_eof(&bytes)?; - - Ok(Script { - id: Uuid::parse_str(&id)?, - name, - description, - shebang, - tags, - script, + rmp::decode::read_total_array(&mut bytes, 6, |b| -> eyre::Result