git-mirrors/continuwuity
1
0
Fork 0
mirror of https://forgejo.ellis.link/continuwuation/continuwuity.git synced 2025-07-05 19:55:49 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Outdent state_compressor service.

Browse source 
Signed-off-by: Jason Volk <jason@zemos.net>
This commit is contained in:
Jason Volk 2025-04-29 06:55:54 +00:00 • committed by Jade Ellis
parent c5c309ec43
commit 56420a67ca
No known key found for this signature in database
GPG key ID: 8705A2A3EBF77BD2
1 changed files with 341 additions and 337 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

128
src/service/rooms/state_compressor/mod.rs
View file

@ -9,7 +9,7 @@ use async_trait::async_trait;
use conduwuit::{
Result,
arrayvec::ArrayVec,
at, checked, err, expected, utils,
at, checked, err, expected, implement, utils,
utils::{bytes, math::usize_from_f64, stream::IterStream},
};
use database::Map;
@ -115,14 +115,14 @@ impl crate::Service for Service {
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
}
impl Service {
/// Returns a stack with info on shortstatehash, full state, added diff and
/// removed diff for the selected shortstatehash and each parent layer.
#[tracing::instrument(name = "load", level = "debug", skip(self))]
pub async fn load_shortstatehash_info(
/// Returns a stack with info on shortstatehash, full state, added diff and
/// removed diff for the selected shortstatehash and each parent layer.
#[implement(Service)]
#[tracing::instrument(name = "load", level = "debug", skip(self))]
pub async fn load_shortstatehash_info(
&self,
shortstatehash: ShortStateHash,
) -> Result<ShortStateInfoVec> {
) -> Result<ShortStateInfoVec> {
if let Some(r) = self.stateinfo_cache.lock()?.get_mut(&shortstatehash) {
return Ok(r.clone());
}
@ -133,11 +133,12 @@ impl Service {
.await?;
Ok(stack)
}
}
/// Returns a stack with info on shortstatehash, full state, added diff and
/// removed diff for the selected shortstatehash and each parent layer.
#[tracing::instrument(
/// Returns a stack with info on shortstatehash, full state, added diff and
/// removed diff for the selected shortstatehash and each parent layer.
#[implement(Service)]
#[tracing::instrument(
name = "cache",
level = "debug",
skip_all,
@ -146,20 +147,21 @@ impl Service {
stack = stack.len(),
),
)]
async fn cache_shortstatehash_info(
async fn cache_shortstatehash_info(
&self,
shortstatehash: ShortStateHash,
stack: ShortStateInfoVec,
) -> Result {
) -> Result {
self.stateinfo_cache.lock()?.insert(shortstatehash, stack);
Ok(())
}
}
async fn new_shortstatehash_info(
#[implement(Service)]
async fn new_shortstatehash_info(
&self,
shortstatehash: ShortStateHash,
) -> Result<ShortStateInfoVec> {
) -> Result<ShortStateInfoVec> {
let StateDiff { parent, added, removed } = self.get_statediff(shortstatehash).await?;
let Some(parent) = parent else {
@ -190,15 +192,16 @@ impl Service {
});
Ok(stack)
}
}
pub fn compress_state_events<'a, I>(
#[implement(Service)]
pub fn compress_state_events<'a, I>(
&'a self,
state: I,
) -> impl Stream<Item = CompressedStateEvent> + Send + 'a
where
) -> impl Stream<Item = CompressedStateEvent> + Send + 'a
where
I: Iterator<Item = (&'a ShortStateKey, &'a EventId)> + Clone + Debug + Send + 'a,
{
{
let event_ids = state.clone().map(at!(1));
let short_event_ids = self
@ -210,16 +213,15 @@ impl Service {
.stream()
.map(at!(0))
.zip(short_event_ids)
.map(|(shortstatekey, shorteventid)| {
compress_state_event(*shortstatekey, shorteventid)
})
}
.map(|(shortstatekey, shorteventid)| compress_state_event(*shortstatekey, shorteventid))
}
pub async fn compress_state_event(
#[implement(Service)]
pub async fn compress_state_event(
&self,
shortstatekey: ShortStateKey,
event_id: &EventId,
) -> CompressedStateEvent {
) -> CompressedStateEvent {
let shorteventid = self
.services
.short
@ -227,34 +229,35 @@ impl Service {
.await;
compress_state_event(shortstatekey, shorteventid)
}
}
/// Creates a new shortstatehash that often is just a diff to an already
/// existing shortstatehash and therefore very efficient.
///
/// There are multiple layers of diffs. The bottom layer 0 always contains
/// the full state. Layer 1 contains diffs to states of layer 0, layer 2
/// diffs to layer 1 and so on. If layer n > 0 grows too big, it will be
/// combined with layer n-1 to create a new diff on layer n-1 that's
/// based on layer n-2. If that layer is also too big, it will recursively
/// fix above layers too.
///
/// * `shortstatehash` - Shortstatehash of this state
/// * `statediffnew` - Added to base. Each vec is shortstatekey+shorteventid
/// * `statediffremoved` - Removed from base. Each vec is
/// shortstatekey+shorteventid
/// * `diff_to_sibling` - Approximately how much the diff grows each time
/// for this layer
/// * `parent_states` - A stack with info on shortstatehash, full state,
/// added diff and removed diff for each parent layer
pub fn save_state_from_diff(
/// Creates a new shortstatehash that often is just a diff to an already
/// existing shortstatehash and therefore very efficient.
///
/// There are multiple layers of diffs. The bottom layer 0 always contains
/// the full state. Layer 1 contains diffs to states of layer 0, layer 2
/// diffs to layer 1 and so on. If layer n > 0 grows too big, it will be
/// combined with layer n-1 to create a new diff on layer n-1 that's
/// based on layer n-2. If that layer is also too big, it will recursively
/// fix above layers too.
///
/// * `shortstatehash` - Shortstatehash of this state
/// * `statediffnew` - Added to base. Each vec is shortstatekey+shorteventid
/// * `statediffremoved` - Removed from base. Each vec is
/// shortstatekey+shorteventid
/// * `diff_to_sibling` - Approximately how much the diff grows each time for
/// this layer
/// * `parent_states` - A stack with info on shortstatehash, full state, added
/// diff and removed diff for each parent layer
#[implement(Service)]
pub fn save_state_from_diff(
&self,
shortstatehash: ShortStateHash,
statediffnew: Arc<CompressedState>,
statediffremoved: Arc<CompressedState>,
diff_to_sibling: usize,
mut parent_states: ParentStatesVec,
) -> Result {
) -> Result {
let statediffnew_len = statediffnew.len();
let statediffremoved_len = statediffremoved.len();
let diffsum = checked!(statediffnew_len + statediffremoved_len)?;
@ -356,16 +359,17 @@ impl Service {
}
Ok(())
}
}
/// Returns the new shortstatehash, and the state diff from the previous
/// room state
#[tracing::instrument(skip(self, new_state_ids_compressed), level = "debug")]
pub async fn save_state(
/// Returns the new shortstatehash, and the state diff from the previous
/// room state
#[implement(Service)]
#[tracing::instrument(skip(self, new_state_ids_compressed), level = "debug")]
pub async fn save_state(
&self,
room_id: &RoomId,
new_state_ids_compressed: Arc<CompressedState>,
) -> Result<HashSetCompressStateEvent> {
) -> Result<HashSetCompressStateEvent> {
let previous_shortstatehash = self
.services
.state
@ -395,8 +399,7 @@ impl Service {
ShortStateInfoVec::new()
};
let (statediffnew, statediffremoved) =
if let Some(parent_stateinfo) = states_parents.last() {
let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() {
let statediffnew: CompressedState = new_state_ids_compressed
.difference(&parent_stateinfo.full_state)
.copied()
@ -428,10 +431,11 @@ impl Service {
added: statediffnew,
removed: statediffremoved,
})
}
}
#[tracing::instrument(skip(self), level = "debug", name = "get")]
async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result<StateDiff> {
#[implement(Service)]
#[tracing::instrument(skip(self), level = "debug", name = "get")]
async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result<StateDiff> {
const BUFSIZE: usize = size_of::<ShortStateHash>();
const STRIDE: usize = size_of::<ShortStateHash>();
@ -475,9 +479,10 @@ impl Service {
added: Arc::new(added),
removed: Arc::new(removed),
})
}
}
fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) {
#[implement(Service)]
fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) {
let mut value = Vec::<u8>::with_capacity(
2_usize
.saturating_add(diff.added.len())
@ -501,7 +506,6 @@ impl Service {
self.db
.shortstatehash_statediff
.insert(&shortstatehash.to_be_bytes(), &value);
}
}
#[inline]