git-mirrors/continuwuity
1
0
Fork 0
mirror of https://forgejo.ellis.link/continuwuation/continuwuity.git synced 2025-07-06 04:16:13 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Outdent state_compressor service.

Browse source 
Signed-off-by: Jason Volk <jason@zemos.net>
This commit is contained in:
Jason Volk 2025-04-29 06:55:54 +00:00 • committed by Jade Ellis
parent c5c309ec43
commit 56420a67ca
No known key found for this signature in database
GPG key ID: 8705A2A3EBF77BD2
1 changed files with 341 additions and 337 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

128
src/service/rooms/state_compressor/mod.rs
View file

@ -9,7 +9,7 @@ use async_trait::async_trait;
use conduwuit::{ use conduwuit::{
Result, Result,
arrayvec::ArrayVec, arrayvec::ArrayVec,
at, checked, err, expected, utils, at, checked, err, expected, implement, utils,
utils::{bytes, math::usize_from_f64, stream::IterStream}, utils::{bytes, math::usize_from_f64, stream::IterStream},
}; };
use database::Map; use database::Map;
@ -115,14 +115,14 @@ impl crate::Service for Service {
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
} }
impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and
/// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer.
/// removed diff for the selected shortstatehash and each parent layer. #[implement(Service)]
#[tracing::instrument(name = "load", level = "debug", skip(self))] #[tracing::instrument(name = "load", level = "debug", skip(self))]
pub async fn load_shortstatehash_info( pub async fn load_shortstatehash_info(
&self, &self,
shortstatehash: ShortStateHash, shortstatehash: ShortStateHash,
) -> Result<ShortStateInfoVec> { ) -> Result<ShortStateInfoVec> {
if let Some(r) = self.stateinfo_cache.lock()?.get_mut(&shortstatehash) { if let Some(r) = self.stateinfo_cache.lock()?.get_mut(&shortstatehash) {
return Ok(r.clone()); return Ok(r.clone());
} }
@ -133,11 +133,12 @@ impl Service {
.await?; .await?;
Ok(stack) Ok(stack)
} }
/// Returns a stack with info on shortstatehash, full state, added diff and /// Returns a stack with info on shortstatehash, full state, added diff and
/// removed diff for the selected shortstatehash and each parent layer. /// removed diff for the selected shortstatehash and each parent layer.
#[tracing::instrument( #[implement(Service)]
#[tracing::instrument(
name = "cache", name = "cache",
level = "debug", level = "debug",
skip_all, skip_all,
@ -146,20 +147,21 @@ impl Service {
stack = stack.len(), stack = stack.len(),
), ),
)] )]
async fn cache_shortstatehash_info( async fn cache_shortstatehash_info(
&self, &self,
shortstatehash: ShortStateHash, shortstatehash: ShortStateHash,
stack: ShortStateInfoVec, stack: ShortStateInfoVec,
) -> Result { ) -> Result {
self.stateinfo_cache.lock()?.insert(shortstatehash, stack); self.stateinfo_cache.lock()?.insert(shortstatehash, stack);
Ok(()) Ok(())
} }
async fn new_shortstatehash_info( #[implement(Service)]
async fn new_shortstatehash_info(
&self, &self,
shortstatehash: ShortStateHash, shortstatehash: ShortStateHash,
) -> Result<ShortStateInfoVec> { ) -> Result<ShortStateInfoVec> {
let StateDiff { parent, added, removed } = self.get_statediff(shortstatehash).await?; let StateDiff { parent, added, removed } = self.get_statediff(shortstatehash).await?;
let Some(parent) = parent else { let Some(parent) = parent else {
@ -190,15 +192,16 @@ impl Service {
}); });
Ok(stack) Ok(stack)
} }
pub fn compress_state_events<'a, I>( #[implement(Service)]
pub fn compress_state_events<'a, I>(
&'a self, &'a self,
state: I, state: I,
) -> impl Stream<Item = CompressedStateEvent> + Send + 'a ) -> impl Stream<Item = CompressedStateEvent> + Send + 'a
where where
I: Iterator<Item = (&'a ShortStateKey, &'a EventId)> + Clone + Debug + Send + 'a, I: Iterator<Item = (&'a ShortStateKey, &'a EventId)> + Clone + Debug + Send + 'a,
{ {
let event_ids = state.clone().map(at!(1)); let event_ids = state.clone().map(at!(1));
let short_event_ids = self let short_event_ids = self
@ -210,16 +213,15 @@ impl Service {
.stream() .stream()
.map(at!(0)) .map(at!(0))
.zip(short_event_ids) .zip(short_event_ids)
.map(|(shortstatekey, shorteventid)| { .map(|(shortstatekey, shorteventid)| compress_state_event(*shortstatekey, shorteventid))
compress_state_event(*shortstatekey, shorteventid) }
})
}
pub async fn compress_state_event( #[implement(Service)]
pub async fn compress_state_event(
&self, &self,
shortstatekey: ShortStateKey, shortstatekey: ShortStateKey,
event_id: &EventId, event_id: &EventId,
) -> CompressedStateEvent { ) -> CompressedStateEvent {
let shorteventid = self let shorteventid = self
.services .services
.short .short
@ -227,34 +229,35 @@ impl Service {
.await; .await;
compress_state_event(shortstatekey, shorteventid) compress_state_event(shortstatekey, shorteventid)
} }
/// Creates a new shortstatehash that often is just a diff to an already /// Creates a new shortstatehash that often is just a diff to an already
/// existing shortstatehash and therefore very efficient. /// existing shortstatehash and therefore very efficient.
/// ///
/// There are multiple layers of diffs. The bottom layer 0 always contains /// There are multiple layers of diffs. The bottom layer 0 always contains
/// the full state. Layer 1 contains diffs to states of layer 0, layer 2 /// the full state. Layer 1 contains diffs to states of layer 0, layer 2
/// diffs to layer 1 and so on. If layer n > 0 grows too big, it will be /// diffs to layer 1 and so on. If layer n > 0 grows too big, it will be
/// combined with layer n-1 to create a new diff on layer n-1 that's /// combined with layer n-1 to create a new diff on layer n-1 that's
/// based on layer n-2. If that layer is also too big, it will recursively /// based on layer n-2. If that layer is also too big, it will recursively
/// fix above layers too. /// fix above layers too.
/// ///
/// * `shortstatehash` - Shortstatehash of this state /// * `shortstatehash` - Shortstatehash of this state
/// * `statediffnew` - Added to base. Each vec is shortstatekey+shorteventid /// * `statediffnew` - Added to base. Each vec is shortstatekey+shorteventid
/// * `statediffremoved` - Removed from base. Each vec is /// * `statediffremoved` - Removed from base. Each vec is
/// shortstatekey+shorteventid /// shortstatekey+shorteventid
/// * `diff_to_sibling` - Approximately how much the diff grows each time /// * `diff_to_sibling` - Approximately how much the diff grows each time for
/// for this layer /// this layer
/// * `parent_states` - A stack with info on shortstatehash, full state, /// * `parent_states` - A stack with info on shortstatehash, full state, added
/// added diff and removed diff for each parent layer /// diff and removed diff for each parent layer
pub fn save_state_from_diff( #[implement(Service)]
pub fn save_state_from_diff(
&self, &self,
shortstatehash: ShortStateHash, shortstatehash: ShortStateHash,
statediffnew: Arc<CompressedState>, statediffnew: Arc<CompressedState>,
statediffremoved: Arc<CompressedState>, statediffremoved: Arc<CompressedState>,
diff_to_sibling: usize, diff_to_sibling: usize,
mut parent_states: ParentStatesVec, mut parent_states: ParentStatesVec,
) -> Result { ) -> Result {
let statediffnew_len = statediffnew.len(); let statediffnew_len = statediffnew.len();
let statediffremoved_len = statediffremoved.len(); let statediffremoved_len = statediffremoved.len();
let diffsum = checked!(statediffnew_len + statediffremoved_len)?; let diffsum = checked!(statediffnew_len + statediffremoved_len)?;
@ -356,16 +359,17 @@ impl Service {
} }
Ok(()) Ok(())
} }
/// Returns the new shortstatehash, and the state diff from the previous /// Returns the new shortstatehash, and the state diff from the previous
/// room state /// room state
#[tracing::instrument(skip(self, new_state_ids_compressed), level = "debug")] #[implement(Service)]
pub async fn save_state( #[tracing::instrument(skip(self, new_state_ids_compressed), level = "debug")]
pub async fn save_state(
&self, &self,
room_id: &RoomId, room_id: &RoomId,
new_state_ids_compressed: Arc<CompressedState>, new_state_ids_compressed: Arc<CompressedState>,
) -> Result<HashSetCompressStateEvent> { ) -> Result<HashSetCompressStateEvent> {
let previous_shortstatehash = self let previous_shortstatehash = self
.services .services
.state .state
@ -395,8 +399,7 @@ impl Service {
ShortStateInfoVec::new() ShortStateInfoVec::new()
}; };
let (statediffnew, statediffremoved) = let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() {
if let Some(parent_stateinfo) = states_parents.last() {
let statediffnew: CompressedState = new_state_ids_compressed let statediffnew: CompressedState = new_state_ids_compressed
.difference(&parent_stateinfo.full_state) .difference(&parent_stateinfo.full_state)
.copied() .copied()
@ -428,10 +431,11 @@ impl Service {
added: statediffnew, added: statediffnew,
removed: statediffremoved, removed: statediffremoved,
}) })
} }
#[tracing::instrument(skip(self), level = "debug", name = "get")] #[implement(Service)]
async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result<StateDiff> { #[tracing::instrument(skip(self), level = "debug", name = "get")]
async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result<StateDiff> {
const BUFSIZE: usize = size_of::<ShortStateHash>(); const BUFSIZE: usize = size_of::<ShortStateHash>();
const STRIDE: usize = size_of::<ShortStateHash>(); const STRIDE: usize = size_of::<ShortStateHash>();
@ -475,9 +479,10 @@ impl Service {
added: Arc::new(added), added: Arc::new(added),
removed: Arc::new(removed), removed: Arc::new(removed),
}) })
} }
fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) { #[implement(Service)]
fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) {
let mut value = Vec::<u8>::with_capacity( let mut value = Vec::<u8>::with_capacity(
2_usize 2_usize
.saturating_add(diff.added.len()) .saturating_add(diff.added.len())
@ -501,7 +506,6 @@ impl Service {
self.db self.db
.shortstatehash_statediff .shortstatehash_statediff
.insert(&shortstatehash.to_be_bytes(), &value); .insert(&shortstatehash.to_be_bytes(), &value);
}
} }
#[inline] #[inline]